/*- * See the file LICENSE for redistribution information. * * Copyright (c) 2002,2010 Oracle. All rights reserved. * * $Id: FileSelectionTest.java,v 1.42.2.4 2010/01/04 15:30:42 cwl Exp $ */ package com.sleepycat.je.cleaner; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Enumeration; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; import com.sleepycat.je.CheckpointConfig; import com.sleepycat.je.Cursor; import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseConfig; import com.sleepycat.je.DatabaseEntry; import com.sleepycat.je.DatabaseException; import com.sleepycat.je.DbInternal; import com.sleepycat.je.DbTestProxy; import com.sleepycat.je.Environment; import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.EnvironmentMutableConfig; import com.sleepycat.je.EnvironmentStats; import com.sleepycat.je.LockMode; import com.sleepycat.je.OperationStatus; import com.sleepycat.je.Transaction; import com.sleepycat.je.cleaner.DbFileSummaryMap; import com.sleepycat.je.config.EnvironmentParams; import com.sleepycat.je.dbi.CursorImpl; import com.sleepycat.je.dbi.DatabaseImpl; import com.sleepycat.je.dbi.EnvironmentImpl; import com.sleepycat.je.dbi.MemoryBudget; import com.sleepycat.je.junit.JUnitThread; import com.sleepycat.je.log.FileManager; import com.sleepycat.je.tree.BIN; import com.sleepycat.je.util.TestUtils; import com.sleepycat.je.utilint.DbLsn; import com.sleepycat.je.utilint.TestHook; public class FileSelectionTest extends TestCase { private static final String DBNAME = "cleanerFileSelection"; private static final int DATA_SIZE = 140; private static final int FILE_SIZE = 4096 * 10; private static final int INITIAL_FILES = 5; private static final int INITIAL_FILES_TEMP = 1; private static final int INITIAL_KEYS = 2000; private static final int INITIAL_KEYS_DUPS = 5000; private static final byte[] MAIN_KEY_FOR_DUPS = {0, 1, 2, 3, 4, 5}; private static final EnvironmentConfig envConfig = initConfig(); private static final EnvironmentConfig highUtilizationConfig = initConfig(); private static final EnvironmentConfig steadyStateAutoConfig = initConfig(); static { highUtilizationConfig.setConfigParam (EnvironmentParams.CLEANER_MIN_UTILIZATION.getName(), String.valueOf(90)); steadyStateAutoConfig.setConfigParam (EnvironmentParams.ENV_RUN_CLEANER.getName(), "true"); } static EnvironmentConfig initConfig() { EnvironmentConfig config = TestUtils.initEnvConfig(); DbInternal.disableParameterValidation(config); config.setTransactional(true); config.setAllowCreate(true); config.setTxnNoSync(Boolean.getBoolean(TestUtils.NO_SYNC)); config.setConfigParam(EnvironmentParams.LOG_FILE_MAX.getName(), Integer.toString(FILE_SIZE)); config.setConfigParam(EnvironmentParams.ENV_CHECK_LEAKS.getName(), "false"); config.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "false"); config.setConfigParam(EnvironmentParams.CLEANER_REMOVE.getName(), "false"); config.setConfigParam (EnvironmentParams.ENV_RUN_CHECKPOINTER.getName(), "false"); config.setConfigParam (EnvironmentParams.CLEANER_LOCK_TIMEOUT.getName(), "1"); config.setConfigParam (EnvironmentParams.CLEANER_MAX_BATCH_FILES.getName(), "1"); return config; } private static final CheckpointConfig forceConfig = new CheckpointConfig(); static { forceConfig.setForce(true); } private File envHome; private Environment env; private EnvironmentImpl envImpl; private Database db; private JUnitThread junitThread; private volatile int synchronizer; private boolean dups; private boolean deferredWrite; private boolean temporary; /* The index is the file number, the value is the first key in the file. */ private List firstKeysInFiles; /* Set of keys that should exist. */ private Set existingKeys; public static Test suite() { TestSuite allTests = new TestSuite(); addTests(allTests, false, false); // transactional addTests(allTests, true, false); // deferredwrite addTests(allTests, false, true); // temporary return allTests; } private static void addTests(TestSuite allTests, boolean deferredWrite, boolean temporary) { TestSuite suite = new TestSuite(FileSelectionTest.class); Enumeration e = suite.tests(); while (e.hasMoreElements()) { FileSelectionTest test = (FileSelectionTest) e.nextElement(); test.init(deferredWrite, temporary); allTests.addTest(test); } } public FileSelectionTest() { envHome = new File(System.getProperty(TestUtils.DEST_DIR)); } private void init(boolean deferredWrite, boolean temporary) { this.deferredWrite = deferredWrite; this.temporary = temporary; } public void setUp() throws IOException, DatabaseException { TestUtils.removeLogFiles("Setup", envHome, false); TestUtils.removeFiles("Setup", envHome, FileManager.DEL_SUFFIX); } public void tearDown() throws IOException, DatabaseException { /* Set test name for reporting; cannot be done in the ctor or setUp. */ setName(getName() + (deferredWrite ? ":deferredWrite" : (temporary ? ":temporary" : ":txnl"))); if (junitThread != null) { while (junitThread.isAlive()) { junitThread.interrupt(); Thread.yield(); } junitThread = null; } try { if (env != null) { env.close(); } } catch (Throwable e) { System.out.println("tearDown: " + e); } //* try { TestUtils.removeLogFiles("tearDown", envHome, true); TestUtils.removeFiles("tearDown", envHome, FileManager.DEL_SUFFIX); } catch (Throwable e) { System.out.println("tearDown: " + e); } //*/ db = null; env = null; envImpl = null; envHome = null; existingKeys = null; firstKeysInFiles = null; } private void openEnv() throws DatabaseException { openEnv(envConfig); } private void openEnv(EnvironmentConfig config) throws DatabaseException { env = new Environment(envHome, config); envImpl = DbInternal.envGetEnvironmentImpl(env); DatabaseConfig dbConfig = new DatabaseConfig(); dbConfig.setTransactional(!isDeferredWriteMode()); dbConfig.setDeferredWrite(deferredWrite); dbConfig.setTemporary(temporary); dbConfig.setAllowCreate(true); dbConfig.setSortedDuplicates(dups); db = env.openDatabase(null, DBNAME, dbConfig); } private void closeEnv() throws DatabaseException { if (temporary) { existingKeys.clear(); } if (db != null) { db.close(); db = null; } if (env != null) { env.close(); env = null; } } /** * Tests that the test utilities work. */ public void testBaseline() throws DatabaseException { int nCleaned; openEnv(); writeData(); verifyData(); nCleaned = cleanRoutine(); /* One file may be cleaned after writing, if a checkpoint occurs. */ assertTrue(String.valueOf(nCleaned), nCleaned <= 1); env.checkpoint(forceConfig); nCleaned = cleanRoutine(); /* One file may be cleaned after cleaning and checkpointing. */ assertTrue(String.valueOf(nCleaned), nCleaned <= 1); closeEnv(); openEnv(); verifyData(); nCleaned = cleanRoutine(); if (temporary) { /* Temp DBs are automaically deleted and cleaned. */ assertTrue(String.valueOf(nCleaned), nCleaned >= INITIAL_FILES_TEMP); } else { /* No files should be cleaned when no writing occurs. */ assertEquals(0, nCleaned); } closeEnv(); } public void testBaselineDups() throws DatabaseException { dups = true; testBaseline(); } /** * Tests that the expected files are selected for cleaning. */ public void testBasic() throws DatabaseException { /* Test assumes that keys are written in order. */ if (isDeferredWriteMode()) { return; } openEnv(); writeData(); verifyDeletedFiles(null); /* * The first file should be the first to be cleaned because it has * relatively few LNs. */ forceCleanOne(); verifyDeletedFiles(new int[] {0}); verifyData(); /* * Delete most of the LNs in two middle files. They should be the next * two files cleaned. */ int fileNum = INITIAL_FILES / 2; int firstKey = ((Integer) firstKeysInFiles.get(fileNum)).intValue(); int nextKey = ((Integer) firstKeysInFiles.get(fileNum + 1)).intValue(); int count = nextKey - firstKey - 4; deleteData(firstKey, count); fileNum += 1; firstKey = ((Integer) firstKeysInFiles.get(fileNum)).intValue(); nextKey = ((Integer) firstKeysInFiles.get(fileNum + 1)).intValue(); count = nextKey - firstKey - 4; deleteData(firstKey, count); forceCleanOne(); forceCleanOne(); verifyDeletedFiles(new int[] {0, fileNum - 1, fileNum}); verifyData(); closeEnv(); } public void testBasicDups() throws DatabaseException { dups = true; testBasic(); } /* * testCleaningMode, testTruncateDatabase, and testRemoveDatabase and are * not tested with dups=true because with duplicates the total utilization * after calling writeData() is 47%, so cleaning will occur and the tests * don't expect that. */ /** * Tests that routine cleaning does not clean when it should not. */ public void testCleaningMode() throws DatabaseException { int nextFile = -1; int nCleaned; /* * Nothing is cleaned with routine cleaning, even after reopening the * environment. */ openEnv(); writeData(); nCleaned = cleanRoutine(); assertEquals(0, nCleaned); nextFile = getNextDeletedFile(nextFile); assertTrue(nextFile == -1); verifyData(); closeEnv(); openEnv(); verifyData(); nCleaned = cleanRoutine(); if (temporary) { assertTrue(String.valueOf(nCleaned), nCleaned >= INITIAL_FILES_TEMP); } else { assertEquals(0, nCleaned); nextFile = getNextDeletedFile(nextFile); assertTrue(nextFile == -1); } verifyData(); closeEnv(); } /** * Test retries after cleaning fails because an LN was write-locked. */ public void testRetry() throws DatabaseException { /* Test assumes that keys are written in order. */ if (isDeferredWriteMode()) { return; } openEnv(highUtilizationConfig); writeData(); verifyData(); /* * The first file is full of LNs. Delete all but the last record to * cause it to be selected next for cleaning. */ int firstKey = ((Integer) firstKeysInFiles.get(1)).intValue(); int nextKey = ((Integer) firstKeysInFiles.get(2)).intValue(); int count = nextKey - firstKey - 1; deleteData(firstKey, count); verifyData(); /* Write-lock the last record to cause cleaning to fail. */ Transaction txn = env.beginTransaction(null, null); Cursor cursor = db.openCursor(txn, null); DatabaseEntry key = new DatabaseEntry(); DatabaseEntry data = new DatabaseEntry(); OperationStatus status; if (dups) { key.setData(MAIN_KEY_FOR_DUPS); data.setData(TestUtils.getTestArray(nextKey - 1)); status = cursor.getSearchBoth(key, data, LockMode.RMW); } else { key.setData(TestUtils.getTestArray(nextKey - 1)); status = cursor.getSearchKey(key, data, LockMode.RMW); } assertEquals(OperationStatus.SUCCESS, status); status = cursor.delete(); assertEquals(OperationStatus.SUCCESS, status); /* Cleaning should fail. */ forceCleanOne(); verifyDeletedFiles(null); forceCleanOne(); verifyDeletedFiles(null); /* Release the write-lock. */ cursor.close(); txn.abort(); verifyData(); /* Cleaning should succeed, with all files deleted. */ forceCleanOne(); verifyDeletedFiles(new int[] {0, 1, 2}); verifyData(); closeEnv(); } /** * Tests that the je.cleaner.minFileUtilization property works as expected. */ public void testMinFileUtilization() throws DatabaseException { /* Test assumes that keys are written in order. */ if (isDeferredWriteMode()) { return; } /* Open with minUtilization=10 and minFileUtilization=0. */ EnvironmentConfig myConfig = initConfig(); myConfig.setConfigParam (EnvironmentParams.CLEANER_MIN_UTILIZATION.getName(), String.valueOf(10)); myConfig.setConfigParam (EnvironmentParams.CLEANER_MIN_FILE_UTILIZATION.getName(), String.valueOf(0)); openEnv(myConfig); /* Write data and delete two thirds of the LNs in the middle file. */ writeData(); verifyDeletedFiles(null); int fileNum = INITIAL_FILES / 2; int firstKey = ((Integer) firstKeysInFiles.get(fileNum)).intValue(); int nextKey = ((Integer) firstKeysInFiles.get(fileNum + 1)).intValue(); int count = ((nextKey - firstKey) * 2) / 3; deleteData(firstKey, count); /* The file should not be deleted. */ env.cleanLog(); env.checkpoint(forceConfig); verifyDeletedFiles(null); /* Change minFileUtilization=50 */ myConfig.setConfigParam (EnvironmentParams.CLEANER_MIN_FILE_UTILIZATION.getName(), String.valueOf(50)); env.setMutableConfig(myConfig); /* The file should now be deleted. */ env.cleanLog(); env.checkpoint(forceConfig); verifyDeletedFiles(new int[] {fileNum}); verifyData(); closeEnv(); } private void printFiles(String msg) { System.out.print(msg); Long lastNum = envImpl.getFileManager().getLastFileNum(); for (int i = 0; i <= (int) lastNum.longValue(); i += 1) { String name = envImpl.getFileManager(). getFullFileName(i, FileManager.JE_SUFFIX); if (new File(name).exists()) { System.out.print(" " + i); } } System.out.println(""); } public void testRetryDups() throws DatabaseException { dups = true; testRetry(); } /** * Steady state should occur with normal (50% utilization) configuration * and automatic checkpointing and cleaning. */ public void testSteadyStateAutomatic() throws DatabaseException { doSteadyState(steadyStateAutoConfig, false, 13); } public void testSteadyStateAutomaticDups() throws DatabaseException { dups = true; testSteadyStateAutomatic(); } /** * Steady state utilization with manual checkpointing and cleaning. */ public void testSteadyStateManual() throws DatabaseException { doSteadyState(envConfig, true, 13); } public void testSteadyStateManualDups() throws DatabaseException { dups = true; testSteadyStateManual(); } /** * Steady state should occur when utilization is at the maximum. */ public void testSteadyStateHighUtilization() throws DatabaseException { /* * I don't know why a larger log is maintained with deferredWrite and * temporary databases. */ doSteadyState(highUtilizationConfig, true, (deferredWrite | temporary) ? 12 : 9); } public void testSteadyStateHighUtilizationDups() throws DatabaseException { dups = true; testSteadyStateHighUtilization(); } /** * Tests that we quickly reach a steady state of disk usage when updates * are made but no net increase in data occurs. * * @param manualCleaning is whether to run cleaning manually every * iteration, or to rely on the cleaner thread. * * @param maxFileCount the maximum number of files allowed for this test. */ private void doSteadyState(EnvironmentConfig config, boolean manualCleaning, int maxFileCount) throws DatabaseException { openEnv(config); writeData(); verifyData(); final int iterations = 100; for (int i = 0; i < iterations; i += 1) { updateData(100, 100); int cleaned = -1; if (manualCleaning) { cleaned = cleanRoutine(); } else { /* Need to delay a bit for the cleaner to keep up. */ try { Thread.sleep(25); } catch (InterruptedException e) {} } /* * Checkpoints need to occur often for the cleaner to keep up. * and to delete files that were cleaned. */ env.checkpoint(forceConfig); verifyData(); int fileCount = envImpl.getFileManager().getAllFileNumbers().length; assertTrue("fileCount=" + fileCount + " maxFileCount=" + maxFileCount + " iteration=" + i, fileCount <= maxFileCount); if (false) { System.out.println("fileCount=" + fileCount + " cleaned=" + cleaned); } } closeEnv(); } /** * Tests that truncate causes cleaning. */ public void testTruncateDatabase() throws IOException, DatabaseException { int nCleaned; openEnv(); writeData(); nCleaned = cleanRoutine(); assertEquals(0, nCleaned); db.close(); db = null; /* * Temporary databases are removed when the database is closed, so * don't call truncate explicitly. */ if (!temporary) { env.truncateDatabase(null, DBNAME, false /* returnCount */); } nCleaned = cleanRoutine(); if (temporary) { assertTrue(String.valueOf(nCleaned), nCleaned >= INITIAL_FILES_TEMP - 1); } else { assertTrue(String.valueOf(nCleaned), nCleaned >= INITIAL_FILES - 1); } closeEnv(); } /** * Tests that remove causes cleaning. */ public void testRemoveDatabase() throws DatabaseException { int nCleaned; openEnv(); writeData(); String dbName = db.getDatabaseName(); db.close(); db = null; nCleaned = cleanRoutine(); if (temporary) { assertTrue(String.valueOf(nCleaned), nCleaned >= INITIAL_FILES_TEMP - 1); assertTrue(!env.getDatabaseNames().contains(dbName)); } else { assertEquals(0, nCleaned); env.removeDatabase(null, dbName); nCleaned = cleanRoutine(); assertTrue(String.valueOf(nCleaned), nCleaned >= INITIAL_FILES - 1); } closeEnv(); } public void testForceCleanFiles() throws DatabaseException { /* When the temp DB is closed many files will be cleaned. */ if (temporary) { return; } /* No files force cleaned. */ EnvironmentConfig myConfig = initConfig(); openEnv(myConfig); writeData(); verifyData(); env.cleanLog(); env.checkpoint(forceConfig); verifyDeletedFiles(null); closeEnv(); /* Force cleaning: 3 */ myConfig.setConfigParam (EnvironmentParams.CLEANER_FORCE_CLEAN_FILES.getName(), "3"); openEnv(myConfig); forceCleanOne(); verifyDeletedFiles(new int[] {3}); closeEnv(); /* Force cleaning: 0 - 1 */ myConfig.setConfigParam (EnvironmentParams.CLEANER_FORCE_CLEAN_FILES.getName(), "0-1"); openEnv(myConfig); forceCleanOne(); forceCleanOne(); verifyDeletedFiles(new int[] {0, 1, 3}); closeEnv(); } /** * Checks that old version log files are upgraded when * je.cleaner.upgradeToLogVersion is set. The version 5 log files to be * upgraded in this test were created with MakeMigrationLogFiles. */ public void testLogVersionUpgrade() throws DatabaseException, IOException { if (temporary) { /* This test is not applicable. */ return; } /* Copy pre-created files 0 and 1, which are log verion 5. */ TestUtils.loadLog (getClass(), "migrate_f0.jdb", envHome, "00000000.jdb"); TestUtils.loadLog (getClass(), "migrate_f1.jdb", envHome, "00000001.jdb"); /* * Write several more files which are log version 6 or greater. To * check whether these files are cleaned below we need to write more * than 2 files (2 is the minimum age for cleaning). */ env = MakeMigrationLogFiles.openEnv(envHome, false /*allowCreate*/); MakeMigrationLogFiles.makeMigrationLogFiles(env); env.checkpoint(forceConfig); MakeMigrationLogFiles.makeMigrationLogFiles(env); env.checkpoint(forceConfig); closeEnv(); /* With upgradeToLogVersion=0 no files should be cleaned. */ openEnvWithUpgradeToLogVersion(0); int nFiles = env.cleanLog(); assertEquals(0, nFiles); closeEnv(); /* With upgradeToLogVersion=5 no files should be cleaned. */ openEnvWithUpgradeToLogVersion(5); nFiles = env.cleanLog(); assertEquals(0, nFiles); closeEnv(); /* Upgrade log files to the current version, which is 6 or greater. */ openEnvWithUpgradeToLogVersion(-1); // -1 means current version /* * Clean one log file at a time so we can check that the backlog is * not impacted by log file migration. */ for (int i = 0; i < 2; i += 1) { nFiles = DbInternal.envGetEnvironmentImpl(env).getCleaner().doClean (false /*cleanMultipleFiles*/, false /*forceCleaning*/); assertEquals(1, nFiles); EnvironmentStats stats = env.getStats(null); assertEquals(0, stats.getCleanerBacklog()); } env.checkpoint(forceConfig); verifyDeletedFiles(new int[] {0, 1}); /* No more files should be cleaned. */ nFiles = env.cleanLog(); assertEquals(0, nFiles); closeEnv(); /* * Force clean file 2 to ensure that it was not cleaned above because * of its log version and not some other factor. */ EnvironmentConfig myConfig = initConfig(); myConfig.setConfigParam (EnvironmentParams.CLEANER_FORCE_CLEAN_FILES.getName(), "2"); openEnv(myConfig); nFiles = env.cleanLog(); assertEquals(1, nFiles); env.checkpoint(forceConfig); verifyDeletedFiles(new int[] {0, 1, 2}); closeEnv(); } private void openEnvWithUpgradeToLogVersion(int upgradeToLogVersion) throws DatabaseException { EnvironmentConfig envConfig = new EnvironmentConfig(); envConfig.setConfigParam (EnvironmentParams.CLEANER_UPGRADE_TO_LOG_VERSION.getName(), String.valueOf(upgradeToLogVersion)); envConfig.setConfigParam (EnvironmentParams.ENV_RUN_CLEANER.getName(), "false"); envConfig.setConfigParam (EnvironmentParams.ENV_RUN_CHECKPOINTER.getName(), "false"); env = new Environment(envHome, envConfig); envImpl = DbInternal.envGetEnvironmentImpl(env); } /** * Tests that when cleaned files are deleted during a compression, the * flushing of the local tracker does not transfer tracker information * for the deleted files. [#15528] * * This test also checks that tracker information is not transfered to the * MapLN's per-DB utilization information in DbFileSummaryMap. This was * occuring in JE 3.3.74 and earlier, under the same circumstances as * tested here (IN compression). [#16610] */ public void testCompressionBug() throws DatabaseException { /* * We need to compress deleted keys and count their utilization under * an explicit compress() call. With deferred write, no utilization * counting occurs until eviction/sync, and that would also do * compression. */ if (isDeferredWriteMode()) { return; } EnvironmentConfig envConfig = initConfig(); /* Disable compressor so we can compress explicitly. */ envConfig.setConfigParam (EnvironmentParams.ENV_RUN_INCOMPRESSOR.getName(), "false"); /* Ensure that we check for resurrected file leaks. */ envConfig.setConfigParam (EnvironmentParams.ENV_CHECK_LEAKS.getName(), "true"); openEnv(envConfig); /* Write and then delete all data. */ writeData(); for (Iterator i = existingKeys.iterator(); i.hasNext();) { int nextKey = ((Integer) i.next()).intValue(); DatabaseEntry key = new DatabaseEntry(TestUtils.getTestArray(nextKey)); OperationStatus status = db.delete(null, key); assertSame(OperationStatus.SUCCESS, status); } synchronizer = 0; /* Create thread that will do the compression. */ junitThread = new JUnitThread("TestCompress") { public void testBody() throws DatabaseException { try { /* compress() will call the test hook below. */ env.compress(); } catch (Throwable e) { e.printStackTrace(); } } }; /* * Set a hook that is called by the INCompressor before it calls * UtilizationProfile.flushLocalTracker. */ envImpl.getINCompressor().setBeforeFlushTrackerHook(new TestHook() { public void doHook() { synchronizer = 1; /* Wait for log cleaning to complete. */ while (synchronizer < 2 && !Thread.interrupted()) { Thread.yield(); } } public Object getHookValue() { throw new UnsupportedOperationException(); } public void doIOHook() throws IOException { throw new UnsupportedOperationException(); } public void hookSetup() { throw new UnsupportedOperationException(); } }); /* Kick off test in thread above. */ junitThread.start(); /* Wait for hook to be called at the end of compression. */ while (synchronizer < 1) Thread.yield(); /* Clean and checkpoint to delete cleaned files. */ while (env.cleanLog() > 0) { } env.checkpoint(forceConfig); /* Allow test hook to return, so that flushLocalTracker is called. */ synchronizer = 2; /* * Before the fix [#15528], an assertion fired in * BaseUtilizationTracker.getFileSummary when flushLocalTracker was * called. This assertion fires if the file being tracked does not * exist. The fix was to check for valid files in flushLocalTracker. */ try { junitThread.finishTest(); junitThread = null; } catch (Throwable e) { e.printStackTrace(); fail(e.toString()); } closeEnv(); } /** * Checks that DB utilization is repaired when damaged by JE 3.3.74 or * earlier. Go to some trouble to create a DatabaseImpl with the repair * done flag not set, and with a DB file summary for a deleted file. * [#16610] */ public void testDbUtilizationRepair() throws DatabaseException, IOException { openEnv(); writeData(); forceCleanOne(); verifyDeletedFiles(new int[] {0}); DatabaseImpl dbImpl = DbInternal.dbGetDatabaseImpl(db); DbFileSummaryMap summaries = dbImpl.getDbFileSummaries(); /* New version DB does not need repair. */ assertTrue(dbImpl.getUtilizationRepairDone()); /* Deleted file is absent from summary map. */ assertNull(summaries.get(0L /*fileNum*/, true /*adjustMemBudget*/, true /*checkResurrected*/, envImpl.getFileManager())); /* * Force addition of deleted file to summary map by creating a dummy * file to prevent assertions from firing. */ File dummyFile = new File(env.getHome(), "00000000.jdb"); assertTrue(dummyFile.createNewFile()); assertNotNull(summaries.get(0L /*fileNum*/, true /*adjustMemBudget*/, false /*checkResurrected*/, envImpl.getFileManager())); assertTrue(dummyFile.delete()); /* Now an entry in the summary map is there for a deleted file.. */ assertNotNull(summaries.get(0L /*fileNum*/, true /*adjustMemBudget*/, true /*checkResurrected*/, envImpl.getFileManager())); /* Force the MapLN with the bad entry to be flushed. */ dbImpl.setDirtyUtilization(); env.checkpoint(forceConfig); closeEnv(); /* If the DB is temporary, we can't test it further. */ if (temporary) { return; } /* * When the DB is opened, the repair should not take place, because we * did not clear the repair done flag above. */ openEnv(); dbImpl = DbInternal.dbGetDatabaseImpl(db); summaries = dbImpl.getDbFileSummaries(); assertTrue(dbImpl.getUtilizationRepairDone()); assertNotNull(summaries.get(0L /*fileNum*/, true /*adjustMemBudget*/, true /*checkResurrected*/, envImpl.getFileManager())); /* Clear the repair done flag and force the MapLN to be flushed. */ dbImpl.clearUtilizationRepairDone(); dbImpl.setDirtyUtilization(); env.checkpoint(forceConfig); closeEnv(); /* * Since the repair done flag was cleared above, when the DB is opened, * the repair should take place. */ openEnv(); dbImpl = DbInternal.dbGetDatabaseImpl(db); summaries = dbImpl.getDbFileSummaries(); assertTrue(dbImpl.getUtilizationRepairDone()); assertNull(summaries.get(0L /*fileNum*/, true /*adjustMemBudget*/, true /*checkResurrected*/, envImpl.getFileManager())); closeEnv(); } /** * Force cleaning of one file. */ private void forceCleanOne() throws DatabaseException { envImpl.getCleaner().doClean(false, // cleanMultipleFiles true); // forceCleaning /* To force file deletion a checkpoint is necessary. */ env.checkpoint(forceConfig); } /** * Do routine cleaning just as normally done via the cleaner daemon, and * return the number of files cleaned. */ private int cleanRoutine() throws DatabaseException { return env.cleanLog(); } /** * Use transactions when not testing deferred write or temporary DBs. */ private boolean isDeferredWriteMode() { return deferredWrite || temporary; } /** * Forces eviction when a temporary database is used, since otherwise data * will not be flushed. */ private void forceEvictionIfTemporary() throws DatabaseException { if (temporary) { EnvironmentMutableConfig config = env.getMutableConfig(); long saveSize = config.getCacheSize(); config.setCacheSize(MemoryBudget.MIN_MAX_MEMORY_SIZE * 2); env.setMutableConfig(config); env.evictMemory(); config.setCacheSize(saveSize); env.setMutableConfig(config); } } /** * Writes data to create INITIAL_FILES number of files, storing the first * key for each file in the firstKeysInFiles list. One extra file is * actually created, to ensure that the firstActiveLSN is not in any of * INITIAL_FILES files. */ private void writeData() throws DatabaseException { int firstFile = (int) envImpl.getFileManager().getLastFileNum().longValue(); assertEquals(0, firstFile); DatabaseEntry key = new DatabaseEntry(); DatabaseEntry data = new DatabaseEntry(new byte[DATA_SIZE]); existingKeys = new HashSet(); if (isDeferredWriteMode()) { firstKeysInFiles = null; Cursor cursor = db.openCursor(null, null); int maxKey = (dups ? INITIAL_KEYS_DUPS : INITIAL_KEYS); for (int nextKey = 0; nextKey < maxKey; nextKey += 1) { OperationStatus status; if (dups) { key.setData(MAIN_KEY_FOR_DUPS); data.setData(TestUtils.getTestArray(nextKey)); status = cursor.putNoDupData(key, data); } else { key.setData(TestUtils.getTestArray(nextKey)); data.setData(new byte[DATA_SIZE]); status = cursor.putNoOverwrite(key, data); } assertEquals(OperationStatus.SUCCESS, status); existingKeys.add(new Integer(nextKey)); } cursor.close(); } else { firstKeysInFiles = new ArrayList(); Transaction txn = env.beginTransaction(null, null); Cursor cursor = db.openCursor(txn, null); int fileNum = -1; for (int nextKey = 0; fileNum < INITIAL_FILES; nextKey += 1) { OperationStatus status; if (dups) { key.setData(MAIN_KEY_FOR_DUPS); data.setData(TestUtils.getTestArray(nextKey)); status = cursor.putNoDupData(key, data); } else { key.setData(TestUtils.getTestArray(nextKey)); data.setData(new byte[DATA_SIZE]); status = cursor.putNoOverwrite(key, data); } assertEquals(OperationStatus.SUCCESS, status); existingKeys.add(new Integer(nextKey)); long lsn = getLsn(cursor); if (DbLsn.getFileNumber(lsn) != fileNum) { assertTrue(fileNum < DbLsn.getFileNumber(lsn)); fileNum = (int) DbLsn.getFileNumber(lsn); assertEquals(fileNum, firstKeysInFiles.size()); firstKeysInFiles.add(new Integer(nextKey)); } } //System.out.println("Num keys: " + existingKeys.size()); cursor.close(); txn.commit(); } forceEvictionIfTemporary(); env.checkpoint(forceConfig); int lastFile = (int) envImpl.getFileManager().getLastFileNum().longValue(); if (temporary) { assertTrue(String.valueOf(lastFile), lastFile >= INITIAL_FILES_TEMP); } else { assertTrue(String.valueOf(lastFile), lastFile >= INITIAL_FILES); } //System.out.println("last file " + lastFile); } /** * Deletes the specified keys. */ private void deleteData(int firstKey, int keyCount) throws DatabaseException { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry data = new DatabaseEntry(); Transaction txn = !isDeferredWriteMode() ? env.beginTransaction(null, null) : null; Cursor cursor = db.openCursor(txn, null); for (int i = 0; i < keyCount; i += 1) { int nextKey = firstKey + i; OperationStatus status; if (dups) { key.setData(MAIN_KEY_FOR_DUPS); data.setData(TestUtils.getTestArray(nextKey)); status = cursor.getSearchBoth(key, data, null); } else { key.setData(TestUtils.getTestArray(nextKey)); status = cursor.getSearchKey(key, data, null); } assertEquals(OperationStatus.SUCCESS, status); status = cursor.delete(); assertEquals(OperationStatus.SUCCESS, status); existingKeys.remove(new Integer(nextKey)); } cursor.close(); if (txn != null) { txn.commit(); } forceEvictionIfTemporary(); } /** * Updates the specified keys. */ private void updateData(int firstKey, int keyCount) throws DatabaseException { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry data = new DatabaseEntry(); Transaction txn = !isDeferredWriteMode() ? env.beginTransaction(null, null) : null; Cursor cursor = db.openCursor(txn, null); for (int i = 0; i < keyCount; i += 1) { int nextKey = firstKey + i; OperationStatus status; if (dups) { key.setData(MAIN_KEY_FOR_DUPS); data.setData(TestUtils.getTestArray(nextKey)); status = cursor.getSearchBoth(key, data, null); assertEquals(OperationStatus.SUCCESS, status); assertEquals(MAIN_KEY_FOR_DUPS.length, key.getSize()); assertEquals(nextKey, TestUtils.getTestVal(data.getData())); } else { key.setData(TestUtils.getTestArray(nextKey)); status = cursor.getSearchKey(key, data, null); assertEquals(OperationStatus.SUCCESS, status); assertEquals(nextKey, TestUtils.getTestVal(key.getData())); assertEquals(DATA_SIZE, data.getSize()); } status = cursor.putCurrent(data); assertEquals(OperationStatus.SUCCESS, status); } cursor.close(); if (txn != null) { txn.commit(); } forceEvictionIfTemporary(); } /** * Verifies that the data written by writeData can be read. */ private void verifyData() throws DatabaseException { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry data = new DatabaseEntry(); Transaction txn = !isDeferredWriteMode() ? env.beginTransaction(null, null) : null; Cursor cursor = db.openCursor(txn, null); for (Iterator i = existingKeys.iterator(); i.hasNext();) { int nextKey = ((Integer) i.next()).intValue(); OperationStatus status; if (dups) { key.setData(MAIN_KEY_FOR_DUPS); data.setData(TestUtils.getTestArray(nextKey)); status = cursor.getSearchBoth(key, data, null); assertEquals(OperationStatus.SUCCESS, status); assertEquals(MAIN_KEY_FOR_DUPS.length, key.getSize()); assertEquals(nextKey, TestUtils.getTestVal(data.getData())); } else { key.setData(TestUtils.getTestArray(nextKey)); status = cursor.getSearchKey(key, data, null); assertEquals(OperationStatus.SUCCESS, status); assertEquals(nextKey, TestUtils.getTestVal(key.getData())); assertEquals(DATA_SIZE, data.getSize()); } } cursor.close(); if (txn != null) { txn.commit(); } } /** * Checks that all log files exist except those specified. */ private void verifyDeletedFiles(int[] shouldNotExist) { Long lastNum = envImpl.getFileManager().getLastFileNum(); for (int i = 0; i <= (int) lastNum.longValue(); i += 1) { boolean shouldExist = true; if (shouldNotExist != null) { for (int j = 0; j < shouldNotExist.length; j += 1) { if (i == shouldNotExist[j]) { shouldExist = false; break; } } } String name = envImpl.getFileManager(). getFullFileName(i, FileManager.JE_SUFFIX); assertEquals(name, shouldExist, new File(name).exists()); } } /** * Returns the first deleted file number or -1 if none. */ private int getNextDeletedFile(int afterFile) { Long lastNum = envImpl.getFileManager().getLastFileNum(); for (int i = afterFile + 1; i <= (int) lastNum.longValue(); i += 1) { String name = envImpl.getFileManager(). getFullFileName(i, FileManager.JE_SUFFIX); if (!(new File(name).exists())) { return i; } } return -1; } /** * Gets the LSN at the cursor position, using internal methods. */ private long getLsn(Cursor cursor) throws DatabaseException { CursorImpl impl = DbTestProxy.dbcGetCursorImpl(cursor); BIN bin; int index; if (dups) { bin = impl.getDupBIN(); index = impl.getDupIndex(); if (bin == null) { bin = impl.getBIN(); index = impl.getIndex(); assertNotNull(bin); } } else { assertNull(impl.getDupBIN()); bin = impl.getBIN(); index = impl.getIndex(); assertNotNull(bin); } assertTrue(index >= 0); long lsn = bin.getLsn(index); assertTrue(lsn != DbLsn.NULL_LSN); return lsn; } }