1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002, 2014 Oracle and/or its affiliates.  All rights reserved.
5  *
6  */
7 
8 package com.sleepycat.je.util;
9 
10 import java.io.BufferedReader;
11 import java.io.File;
12 import java.io.IOException;
13 import java.io.InputStreamReader;
14 import java.text.DecimalFormat;
15 
16 import com.sleepycat.je.CheckpointConfig;
17 import com.sleepycat.je.Cursor;
18 import com.sleepycat.je.Database;
19 import com.sleepycat.je.DatabaseEntry;
20 import com.sleepycat.je.DatabaseException;
21 import com.sleepycat.je.DbInternal;
22 import com.sleepycat.je.Environment;
23 import com.sleepycat.je.EnvironmentConfig;
24 import com.sleepycat.je.EnvironmentMutableConfig;
25 import com.sleepycat.je.EnvironmentStats;
26 import com.sleepycat.je.LockMode;
27 import com.sleepycat.je.OperationStatus;
28 import com.sleepycat.je.StatsConfig;
29 import com.sleepycat.je.Transaction;
30 import com.sleepycat.je.cleaner.VerifyUtils;
31 import com.sleepycat.je.config.EnvironmentParams;
32 import com.sleepycat.je.dbi.EnvironmentImpl;
33 import com.sleepycat.je.utilint.CmdUtil;
34 
35 /**
36  * @hidden
37  * For internal use only.
38  * DbRunAction is a debugging aid that can invoke a JE operation or background
39  * activity from the command line.
40  *
41  *   batchClean calls Environment.cleanLog() in a loop
42  *   checkpoint calls Environment.checkpoint() with force=true
43  *   compress calls Environment.compress
44  *   evict calls Environment.preload, then evictMemory
45  *   removeDb calls Environment.removeDatabase, but doesn't do any cleaning
46  *   removeDbAndClean calls removeDatabase, then cleanLog in a loop
47  *   activateCleaner wakes up the cleaner, and then the main thread waits
48  *     until you type "y" to the console before calling Environment.close().
49  *     The control provided by the prompt is necessary for daemon activities
50  *     because often threads check and bail out if the environment is closed.
51  *   verifyUtilization calls CleanerTestUtils.verifyUtilization() to compare
52  *     utilization as calculated by UtilizationProfile to utilization as
53  *     calculated by UtilizationFileReader.
54  */
55 public class DbRunAction {
56 
57     private static final int BATCH_CLEAN = 1;   // app-driven batch cleaning
58     private static final int COMPRESS = 2;
59     private static final int EVICT = 3;
60     private static final int CHECKPOINT = 4;
61     private static final int REMOVEDB = 5;
62     private static final int REMOVEDB_AND_CLEAN = 6;
63     private static final int ACTIVATE_CLEANER_THREADS = 7;
64                                            // wake up cleaner threads
65     private static final int VERIFY_UTILIZATION = 8;
66 
main(String[] argv)67     public static void main(String[] argv) {
68 
69         long recoveryStart = 0;
70         long actionStart = 0;
71         long actionEnd = 0;
72 
73         try {
74             int whichArg = 0;
75             if (argv.length == 0) {
76                 usage();
77                 System.exit(1);
78             }
79 
80             String dbName = null;
81             int doAction = 0;
82             String envHome = ".";
83             boolean readOnly = false;
84             boolean printStats = false;
85 
86             while (whichArg < argv.length) {
87                 String nextArg = argv[whichArg];
88 
89                 if (nextArg.equals("-h")) {
90                     whichArg++;
91                     envHome = CmdUtil.getArg(argv, whichArg);
92                 } else if (nextArg.equals("-a")) {
93                     whichArg++;
94                     String action = CmdUtil.getArg(argv, whichArg);
95                     if (action.equalsIgnoreCase("batchClean")) {
96                         doAction = BATCH_CLEAN;
97                     } else if (action.equalsIgnoreCase("compress")) {
98                         doAction = COMPRESS;
99                     } else if (action.equalsIgnoreCase("checkpoint")) {
100                         doAction = CHECKPOINT;
101                     } else if (action.equalsIgnoreCase("evict")) {
102                         doAction = EVICT;
103                     } else if (action.equalsIgnoreCase("removedb")) {
104                         doAction = REMOVEDB;
105                     } else if (action.equalsIgnoreCase("removedbAndClean")) {
106                         doAction = REMOVEDB_AND_CLEAN;
107                     } else if (action.equalsIgnoreCase("activateCleaner")) {
108                         doAction = ACTIVATE_CLEANER_THREADS;
109                     } else if (action.equalsIgnoreCase("verifyUtilization")) {
110                         doAction = VERIFY_UTILIZATION;
111                     } else {
112                         usage();
113                         System.exit(1);
114                     }
115                 } else if (nextArg.equals("-ro")) {
116                     readOnly = true;
117                 } else if (nextArg.equals("-s")) {
118                     dbName = argv[++whichArg];
119                 } else if (nextArg.equals("-stats")) {
120                     printStats = true;
121                 } else {
122                     throw new IllegalArgumentException
123                         (nextArg + " is not a supported option.");
124                 }
125                 whichArg++;
126             }
127 
128             EnvironmentConfig envConfig = new EnvironmentConfig();
129 
130             /* Don't debug log to the database log. */
131             if (readOnly) {
132                 envConfig.setConfigParam
133                     (EnvironmentParams.JE_LOGGING_DBLOG.getName(), "false");
134 
135                 envConfig.setReadOnly(true);
136             }
137 
138             /*
139              * If evicting, scan the given database first and don't run the
140              * background evictor.
141              */
142             if (doAction == EVICT) {
143                 envConfig.setConfigParam
144                     (EnvironmentParams.ENV_RUN_EVICTOR.getName(), "false");
145                 envConfig.setConfigParam
146                     (EnvironmentParams.EVICTOR_CRITICAL_PERCENTAGE.getName(),
147                      "1000");
148             }
149 
150             /*
151              * If cleaning, disable the daemon cleaner threads.  The work being
152              * done by these threads is aborted when the environment is closed,
153              * which can result in incomplete log cleaning.
154              */
155             if (doAction == BATCH_CLEAN) {
156                 envConfig.setConfigParam
157                     (EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
158             }
159 
160             recoveryStart = System.currentTimeMillis();
161 
162             Environment env =
163                 new Environment(new File(envHome), envConfig);
164 
165             CheckpointConfig forceConfig = new CheckpointConfig();
166             forceConfig.setForce(true);
167 
168             Thread statsPrinter = null;
169             if (printStats) {
170                 statsPrinter = new StatsPrinter(env);
171                 statsPrinter.start();
172             }
173 
174             boolean promptForShutdown = false;
175             actionStart = System.currentTimeMillis();
176             switch(doAction) {
177             case BATCH_CLEAN:
178                 /* Since this is batch cleaning, repeat until no progress. */
179                 while (true) {
180                     int nFiles = env.cleanLog();
181                     System.out.println("Files cleaned: " + nFiles);
182                     if (nFiles == 0) {
183                         break;
184                     }
185                 }
186                 env.checkpoint(forceConfig);
187                 break;
188             case COMPRESS:
189                 env.compress();
190                 break;
191             case CHECKPOINT:
192                 env.checkpoint(forceConfig);
193                 break;
194             case EVICT:
195                 preload(env, dbName);
196                 break;
197             case REMOVEDB:
198                 removeAndClean(env, dbName, false);
199                 break;
200             case REMOVEDB_AND_CLEAN:
201                 removeAndClean(env, dbName, true);
202                 break;
203             case ACTIVATE_CLEANER_THREADS:
204                 EnvironmentImpl envImpl =
205                     DbInternal.getEnvironmentImpl(env);
206                 envImpl.getCleaner().wakeup();
207                 promptForShutdown = true;
208                 break;
209             case VERIFY_UTILIZATION:
210                 EnvironmentImpl envImpl2 =
211                     DbInternal.getEnvironmentImpl(env);
212                 VerifyUtils. verifyUtilization
213                     (envImpl2,
214                      true,  // expectAccurateObsoleteLNCount
215                      true,  // expectAccurateObsoleteLNSize
216                      true); // expectAccurateDbUtilization
217                 break;
218             }
219             actionEnd = System.currentTimeMillis();
220 
221             if (promptForShutdown) {
222 
223                 /*
224                  * If the requested action is a daemon driven one, we don't
225                  * want the main thread to shutdown the environment until we
226                  * say we're ready
227                  */
228                 waitForShutdown();
229             }
230             if (statsPrinter != null) {
231                 statsPrinter.interrupt();
232                 statsPrinter.join();
233             }
234             env.close();
235         } catch (Exception e) {
236             e.printStackTrace();
237             System.out.println(e.getMessage());
238             usage();
239             System.exit(1);
240         } finally {
241             DecimalFormat f = new DecimalFormat();
242             f.setMaximumFractionDigits(2);
243 
244             long recoveryDuration = actionStart - recoveryStart;
245             System.out.println("\nrecovery time = " +
246                                f.format(recoveryDuration) +
247                                " millis " +
248                                f.format((double)recoveryDuration/60000) +
249                                " minutes");
250 
251             long actionDuration = actionEnd - actionStart;
252             System.out.println("action time = " +
253                                f.format(actionDuration) +
254                                " millis " +
255                                f.format(actionDuration/60000) +
256                                " minutes");
257         }
258     }
259 
removeAndClean(Environment env, String name, boolean doCleaning)260     private static void removeAndClean(Environment env,
261                                        String name,
262                                        boolean doCleaning)
263         throws Exception {
264 
265         long a, c, d, e, f;
266 
267         Transaction txn = null;
268         CheckpointConfig force = new CheckpointConfig();
269         force.setForce(true);
270 
271         a = System.currentTimeMillis();
272         env.removeDatabase(txn, name);
273         c = System.currentTimeMillis();
274 
275         int cleanedCount = 0;
276         if (doCleaning) {
277             while (env.cleanLog() > 0) {
278                 cleanedCount++;
279             }
280         }
281         d = System.currentTimeMillis();
282 
283         System.out.println("cleanedCount=" + cleanedCount);
284         e = 0;
285         f = 0;
286         if (cleanedCount > 0) {
287             e = System.currentTimeMillis();
288             env.checkpoint(force);
289             f = System.currentTimeMillis();
290         }
291 
292         System.out.println("Remove of " + name  +
293                            " remove: " + getSecs(a, c) +
294                            " clean: " + getSecs(c, d) +
295                            " checkpoint: " + getSecs(e, f));
296     }
297 
getSecs(long start, long end)298     private static String getSecs(long start, long end) {
299         return (end-start) / 1000 + " secs";
300     }
301 
preload(Environment env, String dbName)302     private static void preload(Environment env, String dbName)
303         throws Exception {
304 
305         System.out.println("Preload starting");
306         Database db = env.openDatabase(null, dbName, null);
307         Cursor cursor = db.openCursor(null, null);
308         try {
309             DatabaseEntry key = new DatabaseEntry();
310             DatabaseEntry data = new DatabaseEntry();
311             int count = 0;
312             while (cursor.getNext(key, data, LockMode.DEFAULT) ==
313                    OperationStatus.SUCCESS) {
314                 count++;
315                 if ((count % 50000) == 0) {
316                     System.out.println(count + "...");
317                 }
318             }
319             System.out.println("Preloaded " + count + " records");
320         } finally {
321             cursor.close();
322             db.close();
323         }
324     }
325 
326     @SuppressWarnings("unused")
doEvict(Environment env)327     private static void doEvict(Environment env)
328         throws DatabaseException {
329 
330         /* Push the cache size down by half to force eviction. */
331         EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env);
332         long cacheUsage = envImpl.getMemoryBudget().getCacheMemoryUsage();
333         EnvironmentMutableConfig c = new EnvironmentMutableConfig();
334         c.setCacheSize(cacheUsage/2);
335         env.setMutableConfig(c);
336 
337         long start = System.currentTimeMillis();
338         env.evictMemory();
339         long end = System.currentTimeMillis();
340 
341         DecimalFormat f = new DecimalFormat();
342         f.setMaximumFractionDigits(2);
343         System.out.println("evict time=" + f.format(end-start));
344     }
345 
waitForShutdown()346     private static void waitForShutdown()
347         throws IOException {
348 
349         System.out.println
350             ("Wait for daemon activity to run. When ready to stop, type (y)");
351         BufferedReader reader =
352             new BufferedReader(new InputStreamReader(System.in));
353         do {
354             String val = reader.readLine();
355             if (val != null &&
356                 (val.equalsIgnoreCase("y") ||
357                  val.equalsIgnoreCase("yes"))) {
358                 break;
359             } else {
360                 System.out.println("Shutdown? (y)");
361             }
362         } while (true);
363     }
364 
365     private static class StatsPrinter extends Thread {
366 
367         private Environment env;
368 
StatsPrinter(Environment env)369         StatsPrinter(Environment env) {
370             this.env = env;
371         }
372 
373         @Override
run()374         public void run() {
375 
376             StatsConfig clearConfig = new StatsConfig();
377             clearConfig.setClear(true);
378 
379             while (true) {
380                 try {
381                     synchronized (this) {
382                         wait(30 * 1000);
383                     }
384                     EnvironmentStats stats = env.getStats(clearConfig);
385                     System.out.println("\n" + stats + "\n");
386                 } catch (DatabaseException e) {
387                     e.printStackTrace();
388                     break;
389                 } catch (InterruptedException e) {
390                     break;
391                 }
392             }
393         }
394     }
395 
usage()396     private static void usage() {
397         System.out.println("Usage: \n " +
398                            CmdUtil.getJavaCommand(DbRunAction.class));
399         System.out.println("  -h <environment home> ");
400         System.out.println("  -a <batchClean|compress|evict|checkpoint|" +
401                            "removeDb|removeDbAndClean|activateCleaner|" +
402                            "verifyUtilization>");
403         System.out.println("  -ro (read-only - defaults to read-write)");
404         System.out.println("  -s <dbName> (for removeDb)");
405         System.out.println("  -stats (print every 30 seconds)");
406     }
407 }
408