1--[[[
2-- @module lua_maps
3-- This module contains helper functions for managing rspamd maps
4--]]
5
6--[[
7Copyright (c) 2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
8
9Licensed under the Apache License, Version 2.0 (the "License");
10you may not use this file except in compliance with the License.
11You may obtain a copy of the License at
12
13    http://www.apache.org/licenses/LICENSE-2.0
14
15Unless required by applicable law or agreed to in writing, software
16distributed under the License is distributed on an "AS IS" BASIS,
17WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18See the License for the specific language governing permissions and
19limitations under the License.
20]]--
21
22local rspamd_logger = require "rspamd_logger"
23local ts = require("tableshape").types
24local lua_util = require "lua_util"
25
26local exports = {}
27
28local maps_cache = {}
29
30local function map_hash_key(data, mtype)
31  local hash = require "rspamd_cryptobox_hash"
32  local st = hash.create_specific('xxh64')
33  st:update(data)
34  st:update(mtype)
35
36  return st:hex()
37end
38
39local function starts(where,st)
40  return string.sub(where,1,string.len(st))==st
41end
42
43local function cut_prefix(where,st)
44  return string.sub(where,#st + 1)
45end
46
47local function maybe_adjust_type(data,mtype)
48  local function check_prefix(prefix, t)
49    if starts(data, prefix) then
50      data = cut_prefix(data, prefix)
51      mtype = t
52
53      return true
54    end
55
56    return false
57  end
58
59  local known_types = {
60    {'regexp;', 'regexp'},
61    {'re;', 'regexp'},
62    {'regexp_multi;', 'regexp_multi'},
63    {'re_multi;', 'regexp_multi'},
64    {'glob;', 'glob'},
65    {'glob_multi;', 'glob_multi'},
66    {'radix;', 'radix'},
67    {'ipnet;', 'radix'},
68    {'set;', 'set'},
69    {'hash;', 'hash'},
70    {'plain;', 'hash'},
71    {'cdb;', 'cdb'},
72    {'cdb:/', 'cdb'},
73  }
74
75  if mtype == 'callback' then
76    return mtype
77  end
78
79  for _,t in ipairs(known_types) do
80    if check_prefix(t[1], t[2]) then
81      return data,mtype
82    end
83  end
84
85  -- No change
86  return data,mtype
87end
88
89--[[[
90-- @function lua_maps.map_add_from_ucl(opt, mtype, description)
91-- Creates a map from static data
92-- Returns true if map was added or nil
93-- @param {string or table} opt data for map (or URL)
94-- @param {string} mtype type of map (`set`, `map`, `radix`, `regexp`)
95-- @param {string} description human-readable description of map
96-- @return {bool} true on success, or `nil`
97--]]
98
99local function rspamd_map_add_from_ucl(opt, mtype, description)
100  local ret = {
101    get_key = function(t, k)
102      if t.__data then
103        return t.__data:get_key(k)
104      end
105
106      return nil
107    end
108  }
109  local ret_mt = {
110    __index = function(t, k)
111      if t.__data then
112        return t.get_key(k)
113      end
114
115      return nil
116    end
117  }
118
119  if not opt then
120    return nil
121  end
122
123  if type(opt) == 'string' then
124    opt,mtype = maybe_adjust_type(opt, mtype)
125    local cache_key = map_hash_key(opt, mtype)
126    if maps_cache[cache_key] then
127      rspamd_logger.infox(rspamd_config, 'reuse url for %s(%s)',
128          opt, mtype)
129
130      return maps_cache[cache_key]
131    end
132    -- We have a single string, so we treat it as a map
133    local map = rspamd_config:add_map{
134      type = mtype,
135      description = description,
136      url = opt,
137    }
138
139    if map then
140      ret.__data = map
141      ret.hash = cache_key
142      setmetatable(ret, ret_mt)
143      maps_cache[cache_key] = ret
144      return ret
145    end
146  elseif type(opt) == 'table' then
147    local cache_key = lua_util.table_digest(opt)
148    if maps_cache[cache_key] then
149      rspamd_logger.infox(rspamd_config, 'reuse url for complex map definition %s: %s',
150          cache_key:sub(1,8), description)
151
152      return maps_cache[cache_key]
153    end
154
155    if opt[1] then
156      -- Adjust each element if needed
157      local adjusted
158      for i,source in ipairs(opt) do
159        local nsrc,ntype = maybe_adjust_type(source, mtype)
160
161        if mtype ~= ntype then
162          if not adjusted then
163            mtype = ntype
164          end
165          adjusted = true
166        end
167        opt[i] = nsrc
168      end
169
170      if mtype == 'radix' then
171
172        if string.find(opt[1], '^%d') then
173          local map = rspamd_config:radix_from_ucl(opt)
174
175          if map then
176            ret.__data = map
177            setmetatable(ret, ret_mt)
178            maps_cache[cache_key] = ret
179            return ret
180          end
181        else
182          -- Plain table
183          local map = rspamd_config:add_map{
184            type = mtype,
185            description = description,
186            url = opt,
187          }
188          if map then
189            ret.__data = map
190            setmetatable(ret, ret_mt)
191            maps_cache[cache_key] = ret
192            return ret
193          end
194        end
195      elseif mtype == 'regexp' or mtype == 'glob' then
196        if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then
197          -- Plain table
198          local map = rspamd_config:add_map{
199            type = mtype,
200            description = description,
201            url = opt,
202          }
203          if map then
204            ret.__data = map
205            setmetatable(ret, ret_mt)
206            maps_cache[cache_key] = ret
207            return ret
208          end
209        else
210          local map = rspamd_config:add_map{
211            type = mtype,
212            description = description,
213            url = {
214              url = 'static',
215              data = opt,
216            }
217          }
218          if map then
219            ret.__data = map
220            setmetatable(ret, ret_mt)
221            maps_cache[cache_key] = ret
222            return ret
223          end
224        end
225      else
226        if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then
227          -- Plain table
228          local map = rspamd_config:add_map{
229            type = mtype,
230            description = description,
231            url = opt,
232          }
233          if map then
234            ret.__data = map
235            setmetatable(ret, ret_mt)
236            maps_cache[cache_key] = ret
237            return ret
238          end
239        else
240          local data = {}
241          local nelts = 0
242          -- Plain array of keys, count merely numeric elts
243          for _,elt in ipairs(opt) do
244            if type(elt) == 'string' then
245              -- Numeric table
246              if mtype == 'hash' then
247                -- Treat as KV pair
248                local pieces = lua_util.str_split(elt, ' ')
249                if #pieces > 1 then
250                  local key = table.remove(pieces, 1)
251                  data[key] = table.concat(pieces, ' ')
252                else
253                  data[elt] = true
254                end
255              else
256                data[elt] = true
257              end
258
259              nelts = nelts + 1
260            end
261          end
262
263          if nelts > 0 then
264            -- Plain Lua table that is used as a map
265            ret.__data = data
266            ret.get_key = function(t, k)
267              if k ~= '__data' then
268                return t.__data[k]
269              end
270
271              return nil
272            end
273
274            maps_cache[cache_key] = ret
275            return ret
276          else
277            -- Empty map, huh?
278            rspamd_logger.errx(rspamd_config, 'invalid map element: %s',
279                opt)
280          end
281        end
282      end
283    else
284      -- We have some non-trivial object so let C code to deal with it somehow...
285      local map = rspamd_config:add_map{
286        type = mtype,
287        description = description,
288        url = opt,
289      }
290      if map then
291        ret.__data = map
292        setmetatable(ret, ret_mt)
293        maps_cache[cache_key] = ret
294        return ret
295      end
296    end -- opt[1]
297  end
298
299  return nil
300end
301
302--[[[
303-- @function lua_maps.map_add(mname, optname, mtype, description)
304-- Creates a map from configuration elements (static data or URL)
305-- Returns true if map was added or nil
306-- @param {string} mname config section to use
307-- @param {string} optname option name to use
308-- @param {string} mtype type of map ('set', 'hash', 'radix', 'regexp', 'glob')
309-- @param {string} description human-readable description of map
310-- @return {bool} true on success, or `nil`
311--]]
312
313local function rspamd_map_add(mname, optname, mtype, description)
314  local opt = rspamd_config:get_module_opt(mname, optname)
315
316  return rspamd_map_add_from_ucl(opt, mtype, description)
317end
318
319exports.rspamd_map_add = rspamd_map_add
320exports.map_add = rspamd_map_add
321exports.rspamd_map_add_from_ucl = rspamd_map_add_from_ucl
322exports.map_add_from_ucl = rspamd_map_add_from_ucl
323
324-- Check `what` for being lua_map name, otherwise just compares key with what
325local function rspamd_maybe_check_map(key, what)
326  local fun = require "fun"
327
328  if type(what) == "table" then
329    return fun.any(function(elt) return rspamd_maybe_check_map(key, elt) end, what)
330  end
331  if type(rspamd_maps) == "table" then
332    local mn
333    if starts(what, "map:") then
334      mn = string.sub(what, 4)
335    elseif starts(what, "map://") then
336      mn = string.sub(what, 6)
337    end
338
339    if mn and rspamd_maps[mn] then
340      return rspamd_maps[mn]:get_key(key)
341    else
342      return what:lower() == key
343    end
344  else
345    return what:lower() == key
346  end
347
348end
349
350exports.rspamd_maybe_check_map = rspamd_maybe_check_map
351
352--[[[
353-- @function lua_maps.fill_config_maps(mname, options, defs)
354-- Fill maps that could be defined in defs, from the config in the options
355-- Defs is a table indexed by a map's parameter name and defining it's config,
356-- for example:
357defs = {
358  my_map = {
359    type = 'map',
360    description = 'my cool map',
361    optional = true,
362  }
363}
364-- Then this function will look for opts.my_map parameter and try to replace it's with
365-- a map with the specific type, description but not failing if it was empty.
366-- It will also set options.my_map_orig to the original value defined in the map
367--]]
368exports.fill_config_maps = function(mname, opts, map_defs)
369  assert(type(opts) == 'table')
370  assert(type(map_defs) == 'table')
371  for k, v in pairs(map_defs) do
372    if opts[k] then
373      local map = rspamd_map_add_from_ucl(opts[k], v.type or 'map', v.description)
374      if not map then
375        rspamd_logger.errx(rspamd_config, 'map add error %s for module %s', k, mname)
376        return false
377      end
378      opts[k..'_orig'] = opts[k]
379      opts[k] = map
380    elseif not v.optional then
381      rspamd_logger.errx(rspamd_config, 'cannot find non optional map %s for module %s', k, mname)
382      return false
383    end
384  end
385
386  return true
387end
388
389exports.map_schema = ts.one_of{
390  ts.string, -- 'http://some_map'
391  ts.array_of(ts.string), -- ['foo', 'bar']
392  ts.shape{ -- complex object
393    name = ts.string:is_optional(),
394    description = ts.string:is_optional(),
395    timeout = ts.number,
396    data = ts.array_of(ts.string):is_optional(),
397    -- Tableshape has no options support for something like key1 or key2?
398    upstreams = ts.one_of{
399      ts.string,
400      ts.array_of(ts.string),
401    }:is_optional(),
402    url = ts.one_of{
403      ts.string,
404      ts.array_of(ts.string),
405    }:is_optional(),
406  }
407}
408
409return exports
410