1defmodule Mix.Tasks.Gettext.Merge do
2  use Mix.Task
3  @recursive true
4
5  @shortdoc "Merge template files into translation files"
6
7  @moduledoc """
8  Merges PO/POT files with PO files.
9
10  This task is used when translations in the source code change: when they do,
11  `mix gettext.extract` is usually used to extract the new translations to POT
12  files. At this point, developers or translators can use this task to "sync"
13  the newly updated POT files with the existing locale-specific PO files. All
14  the metadata for each translation (like position in the source code, comments
15  and so on) is taken from the newly updated POT file; the only things taken
16  from the PO file are the actual translated strings.
17
18  #### Fuzzy matching
19
20  Translations in the updated PO/POT file that have an exact match (a
21  translation with the same msgid) in the old PO file are merged as described
22  above. When a translation in the update PO/POT files has no match in the old
23  PO file, a fuzzy match for that translation is attempted. For example, assume
24  we have this POT file:
25
26      msgid "hello, world!"
27      msgstr ""
28
29  and we merge it with this PO file:
30
31      # notice no exclamation point here
32      msgid "hello, world"
33      msgstr "ciao, mondo"
34
35  Since the two translations are very similar, the msgstr from the existing
36  translation will be taken over to the new translation, which will however be
37  marked as *fuzzy*:
38
39      #, fuzzy
40      msgid "hello, world!"
41      msgstr "ciao, mondo"
42
43  Generally, a `fuzzy` flag calls for review from a translator.
44
45  Fuzzy matching can be configured (for example, the threshold for translation
46  similarity can be tweaked) or disabled entirely; lool at the "Options" section
47  below.
48
49  ## Usage
50
51      mix gettext.merge OLD_FILE UPDATED_FILE [OPTIONS]
52      mix gettext.merge DIR [OPTIONS]
53
54  If two files are given as arguments, they must be a `.po` file and a
55  `.po`/`.pot` file. The first one is the old PO file, while the second one is
56  the last generated one. They are merged and written over the first file. For
57  example:
58
59      mix gettext.merge priv/gettext/en/LC_MESSAGES/default.po priv/gettext/default.pot
60
61  If only one argument is given, then that argument must be a directory
62  containing Gettext translations (with `.pot` files at the root level alongside
63  locale directories - this is usually a "backend" directory used by a Gettext
64  backend, see `Gettext.Backend`).
65
66      mix gettext.merge priv/gettext
67
68  If the `--locale LOCALE` option is given, then only the PO files in
69  `DIR/LOCALE/LC_MESSAGES` will be merged with the POT files in `DIR`. If no
70  options are given, then all the PO files for all locales under `DIR` are
71  merged with the POT files in `DIR`.
72
73  ## Plural forms
74
75  By default, Gettext will determine the number of plural forms for newly generated translations
76  by checking the value of `nplurals` in the `Plural-Forms` header in the existing `.po` file. If
77  a `.po` file doesn't already exist and Gettext is creating a new one or if the `Plural-Forms`
78  header is not in the `.po` file, Gettext will use the number of plural forms that
79  `Gettext.Plural` returns for the locale of the file being created. The number of plural forms
80  can be forced through the `--plural-forms` option (see below).
81
82  ## Options
83
84    * `--locale` - a string representing a locale. If this is provided, then only the PO
85      files in `DIR/LOCALE/LC_MESSAGES` will be merged with the POT files in `DIR`. This
86      option can only be given when a single argument is passed to the task
87      (a directory).
88
89    * `--no-fuzzy` - stops fuzzy matching from being performed when merging
90      files.
91
92    * `--fuzzy-threshold` - a float between `0` and `1` which represents the
93      miminum Jaro distance needed for two translations to be considered a fuzzy
94      match. Overrides the global `:fuzzy_threshold` option (see the docs for
95      `Gettext` for more information on this option).
96
97    * `--plural-forms` - a integer strictly greater than `0`. If this is passed,
98      new translations in the target PO files will have this number of empty
99      plural forms. See the "Plural forms" section above.
100
101  """
102
103  @default_fuzzy_threshold 0.8
104  @switches [
105    locale: :string,
106    fuzzy: :boolean,
107    fuzzy_threshold: :float,
108    plural_forms: :integer
109  ]
110
111  alias Gettext.{Merger, PO}
112
113  def run(args) do
114    _ = Mix.Project.get!()
115    gettext_config = Mix.Project.config()[:gettext] || []
116
117    case OptionParser.parse!(args, switches: @switches) do
118      {opts, [po_file, reference_file]} ->
119        merge_two_files(po_file, reference_file, opts, gettext_config)
120
121      {opts, [translations_dir]} ->
122        merge_translations_dir(translations_dir, opts, gettext_config)
123
124      {_opts, []} ->
125        Mix.raise(
126          "gettext.merge requires at least one argument to work. " <>
127            "Use `mix help gettext.merge` to see the usage of this task"
128        )
129
130      {_opts, _args} ->
131        Mix.raise(
132          "Too many arguments for the gettext.merge task. " <>
133            "Use `mix help gettext.merge` to see the usage of this task"
134        )
135    end
136
137    Mix.Task.reenable("gettext.merge")
138  end
139
140  defp merge_two_files(po_file, reference_file, opts, gettext_config) do
141    merging_opts = validate_merging_opts!(opts, gettext_config)
142
143    if Path.extname(po_file) == ".po" and Path.extname(reference_file) in [".po", ".pot"] do
144      ensure_file_exists!(po_file)
145      ensure_file_exists!(reference_file)
146      locale = locale_from_path(po_file)
147      contents = merge_files(po_file, reference_file, locale, merging_opts, gettext_config)
148      write_file(po_file, contents)
149    else
150      Mix.raise("Arguments must be a PO file and a PO/POT file")
151    end
152  end
153
154  defp merge_translations_dir(dir, opts, gettext_config) do
155    ensure_dir_exists!(dir)
156    merging_opts = validate_merging_opts!(opts, gettext_config)
157
158    if locale = opts[:locale] do
159      merge_locale_dir(dir, locale, merging_opts, gettext_config)
160    else
161      merge_all_locale_dirs(dir, merging_opts, gettext_config)
162    end
163  end
164
165  defp merge_locale_dir(pot_dir, locale, opts, gettext_config) do
166    locale_dir = locale_dir(pot_dir, locale)
167    create_missing_locale_dir(locale_dir)
168    merge_dirs(locale_dir, pot_dir, locale, opts, gettext_config)
169  end
170
171  defp merge_all_locale_dirs(pot_dir, opts, gettext_config) do
172    for locale <- File.ls!(pot_dir), File.dir?(Path.join(pot_dir, locale)) do
173      merge_dirs(locale_dir(pot_dir, locale), pot_dir, locale, opts, gettext_config)
174    end
175  end
176
177  def locale_dir(pot_dir, locale) do
178    Path.join([pot_dir, locale, "LC_MESSAGES"])
179  end
180
181  defp merge_dirs(po_dir, pot_dir, locale, opts, gettext_config) do
182    merger = fn pot_file ->
183      po_file = find_matching_po(pot_file, po_dir)
184      contents = merge_or_create(pot_file, po_file, locale, opts, gettext_config)
185      write_file(po_file, contents)
186    end
187
188    pot_dir
189    |> Path.join("*.pot")
190    |> Path.wildcard()
191    |> Task.async_stream(merger, ordered: false, timeout: 10_000)
192    |> Stream.run()
193
194    warn_for_po_without_pot(po_dir, pot_dir)
195  end
196
197  defp find_matching_po(pot_file, po_dir) do
198    domain = Path.basename(pot_file, ".pot")
199    Path.join(po_dir, "#{domain}.po")
200  end
201
202  defp merge_or_create(pot_file, po_file, locale, opts, gettext_config) do
203    if File.regular?(po_file) do
204      merge_files(po_file, pot_file, locale, opts, gettext_config)
205    else
206      Merger.new_po_file(po_file, pot_file, locale, opts, gettext_config)
207    end
208  end
209
210  defp merge_files(po_file, pot_file, locale, opts, gettext_config) do
211    merged = Merger.merge(PO.parse_file!(po_file), PO.parse_file!(pot_file), locale, opts)
212    PO.dump(merged, gettext_config)
213  end
214
215  defp write_file(path, contents) do
216    File.write!(path, contents)
217    Mix.shell().info("Wrote #{path}")
218  end
219
220  # Warns for every PO file that has no matching POT file.
221  defp warn_for_po_without_pot(po_dir, pot_dir) do
222    po_dir
223    |> Path.join("*.po")
224    |> Path.wildcard()
225    |> Enum.reject(&po_has_matching_pot?(&1, pot_dir))
226    |> Enum.each(fn po_file ->
227      Mix.shell().info("Warning: PO file #{po_file} has no matching POT file in #{pot_dir}")
228    end)
229  end
230
231  defp po_has_matching_pot?(po_file, pot_dir) do
232    domain = Path.basename(po_file, ".po")
233    pot_path = Path.join(pot_dir, "#{domain}.pot")
234    File.exists?(pot_path)
235  end
236
237  defp ensure_file_exists!(path) do
238    unless File.regular?(path), do: Mix.raise("No such file: #{path}")
239  end
240
241  defp ensure_dir_exists!(path) do
242    unless File.dir?(path), do: Mix.raise("No such directory: #{path}")
243  end
244
245  defp create_missing_locale_dir(dir) do
246    unless File.dir?(dir) do
247      File.mkdir_p!(dir)
248      Mix.shell().info("Created directory #{dir}")
249    end
250  end
251
252  defp validate_merging_opts!(opts, gettext_config) do
253    opts =
254      opts
255      |> Keyword.take([:fuzzy, :fuzzy_threshold, :plural_forms])
256      |> Keyword.put_new(:fuzzy, true)
257      |> Keyword.put_new_lazy(:fuzzy_threshold, fn ->
258        gettext_config[:fuzzy_threshold] || @default_fuzzy_threshold
259      end)
260
261    threshold = opts[:fuzzy_threshold]
262
263    unless threshold >= 0.0 and threshold <= 1.0 do
264      Mix.raise("The :fuzzy_threshold option must be a float >= 0.0 and <= 1.0")
265    end
266
267    opts
268  end
269
270  defp locale_from_path(path) do
271    parts = Path.split(path)
272    index = Enum.find_index(parts, &(&1 == "LC_MESSAGES"))
273    Enum.at(parts, index - 1)
274  end
275end
276