1<?php
2/**
3 * Matomo - free/libre analytics platform
4 *
5 * @link https://matomo.org
6 * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
7 *
8 */
9namespace Piwik\Plugins\CoreAdminHome\Commands;
10
11use Piwik\Archive;
12use Piwik\Archive\ArchivePurger;
13use Piwik\DataAccess\ArchiveTableCreator;
14use Piwik\Date;
15use Piwik\Db;
16use Piwik\Plugin\ConsoleCommand;
17use Piwik\Timer;
18use Psr\Log\NullLogger;
19use Symfony\Component\Console\Input\InputArgument;
20use Symfony\Component\Console\Input\InputInterface;
21use Symfony\Component\Console\Input\InputOption;
22use Symfony\Component\Console\Output\OutputInterface;
23
24/**
25 * Command that allows users to force purge old or invalid archive data. In the event of a failure
26 * in the archive purging scheduled task, this command can be used to manually delete old/invalid archives.
27 */
28class PurgeOldArchiveData extends ConsoleCommand
29{
30    const ALL_DATES_STRING = 'all';
31
32    /**
33     * For tests.
34     *
35     * @var Date
36     */
37    public static $todayOverride = null;
38
39    /**
40     * @var ArchivePurger
41     */
42    private $archivePurger;
43
44    public function __construct(ArchivePurger $archivePurger = null)
45    {
46        parent::__construct();
47
48        $this->archivePurger = $archivePurger;
49    }
50
51    protected function configure()
52    {
53        $this->setName('core:purge-old-archive-data');
54        $this->setDescription('Purges out of date and invalid archive data from archive tables.');
55        $this->addArgument("dates", InputArgument::IS_ARRAY | InputArgument::OPTIONAL,
56            "The months of the archive tables to purge data from. By default, only deletes from the current month. Use '" . self::ALL_DATES_STRING. "' for all dates.",
57            array(self::getToday()->toString()));
58        $this->addOption('exclude-outdated', null, InputOption::VALUE_NONE, "Do not purge outdated archive data.");
59        $this->addOption('exclude-invalidated', null, InputOption::VALUE_NONE, "Do not purge invalidated archive data.");
60        $this->addOption('exclude-ranges', null, InputOption::VALUE_NONE, "Do not purge custom ranges.");
61        $this->addOption('skip-optimize-tables', null, InputOption::VALUE_NONE, "Do not run OPTIMIZE TABLES query on affected archive tables.");
62        $this->addOption('include-year-archives', null, InputOption::VALUE_NONE, "If supplied, the command will purge archive tables that contain year archives for every supplied date.");
63        $this->setHelp("By default old and invalidated archives are purged. Custom ranges are also purged with outdated archives.\n\n"
64                     . "Note: archive purging is done during scheduled task execution, so under normal circumstances, you should not need to "
65                     . "run this command manually.");
66
67    }
68
69    protected function execute(InputInterface $input, OutputInterface $output)
70    {
71        // during normal command execution, we don't want the INFO level logs logged by the ArchivePurger service
72        // to display in the console, so we use a NullLogger for the service
73        $logger = null;
74        if ($output->getVerbosity() <= OutputInterface::VERBOSITY_NORMAL) {
75            $logger = new NullLogger();
76        }
77
78        $archivePurger = $this->archivePurger ?: new ArchivePurger($model = null, $purgeDatesOlderThan = null, $logger);
79
80        $dates = $this->getDatesToPurgeFor($input);
81
82        $excludeOutdated = $input->getOption('exclude-outdated');
83        if ($excludeOutdated) {
84            $output->writeln("Skipping purge outdated archive data.");
85        } else {
86            foreach ($dates as $date) {
87                $message = sprintf("Purging outdated archives for %s...", $date->toString('Y_m'));
88                $this->performTimedPurging($output, $message, function () use ($date, $archivePurger) {
89                    $archivePurger->purgeOutdatedArchives($date);
90                });
91            }
92        }
93
94        $excludeInvalidated = $input->getOption('exclude-invalidated');
95        if ($excludeInvalidated) {
96            $output->writeln("Skipping purge invalidated archive data.");
97        } else {
98            foreach ($dates as $date) {
99                $message = sprintf("Purging invalidated archives for %s...", $date->toString('Y_m'));
100                $this->performTimedPurging($output, $message, function () use ($archivePurger, $date) {
101                    $archivePurger->purgeInvalidatedArchivesFrom($date);
102                });
103            }
104        }
105
106        $excludeCustomRanges = $input->getOption('exclude-ranges');
107        if ($excludeCustomRanges) {
108            $output->writeln("Skipping purge custom range archives.");
109        } else {
110            foreach ($dates as $date) {
111                $message = sprintf("Purging custom range archives for %s...", $date->toString('Y_m'));
112                $this->performTimedPurging($output, $message, function () use ($date, $archivePurger) {
113                    $archivePurger->purgeArchivesWithPeriodRange($date);
114                });
115            }
116        }
117
118        $skipOptimizeTables = $input->getOption('skip-optimize-tables');
119        if ($skipOptimizeTables) {
120            $output->writeln("Skipping OPTIMIZE TABLES.");
121        } else {
122            $this->optimizeArchiveTables($output, $dates);
123        }
124    }
125
126    /**
127     * @param InputInterface $input
128     * @return Date[]
129     */
130    private function getDatesToPurgeFor(InputInterface $input)
131    {
132        $dates = array();
133
134        $dateSpecifier = $input->getArgument('dates');
135        if (count($dateSpecifier) === 1
136            && reset($dateSpecifier) == self::ALL_DATES_STRING
137        ) {
138            foreach (ArchiveTableCreator::getTablesArchivesInstalled() as $table) {
139                $tableDate = ArchiveTableCreator::getDateFromTableName($table);
140
141                list($year, $month) = explode('_', $tableDate);
142
143                try {
144                    $date    = Date::factory($year . '-' . $month . '-' . '01');
145                    $dates[] = $date;
146                } catch (\Exception $e) {
147                    // this might occur if archive tables like piwik_archive_numeric_1875_09 exist
148                }
149            }
150        } else {
151            $includeYearArchives = $input->getOption('include-year-archives');
152
153            foreach ($dateSpecifier as $date) {
154                $dateObj = Date::factory($date);
155                $yearMonth = $dateObj->toString('Y-m');
156                $dates[$yearMonth] = $dateObj;
157
158                // if --include-year-archives is supplied, add a date for the january table for this date's year
159                // so year archives will be purged
160                if ($includeYearArchives) {
161                    $janYearMonth = $dateObj->toString('Y') . '-01';
162                    if (empty($dates[$janYearMonth])) {
163                        $dates[$janYearMonth] = Date::factory($janYearMonth . '-01');
164                    }
165                }
166            }
167
168            $dates = array_values($dates);
169        }
170
171        return $dates;
172    }
173
174    private function performTimedPurging(OutputInterface $output, $startMessage, $callback)
175    {
176        $timer = new Timer();
177
178        $output->write($startMessage);
179
180        $callback();
181
182        $output->writeln("Done. <comment>[" . $timer->__toString() . "]</comment>");
183    }
184
185    /**
186     * @param OutputInterface $output
187     * @param Date[] $dates
188     */
189    private function optimizeArchiveTables(OutputInterface $output, $dates)
190    {
191        $output->writeln("Optimizing archive tables...");
192
193        foreach ($dates as $date) {
194            $numericTable = ArchiveTableCreator::getNumericTable($date);
195            $this->performTimedPurging($output, "Optimizing table $numericTable...", function () use ($numericTable) {
196                Db::optimizeTables($numericTable, $force = true);
197            });
198
199            $blobTable = ArchiveTableCreator::getBlobTable($date);
200            $this->performTimedPurging($output, "Optimizing table $blobTable...", function () use ($blobTable) {
201                Db::optimizeTables($blobTable, $force = true);
202            });
203        }
204    }
205
206    private static function getToday()
207    {
208        return self::$todayOverride ?: Date::today();
209    }
210}