/*- * See the file LICENSE for redistribution information. * * Copyright (c) 2002,2010 Oracle. All rights reserved. * * $Id: UtilizationTest.java,v 1.28.2.2 2010/01/04 15:30:42 cwl Exp $ */ package com.sleepycat.je.cleaner; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Enumeration; import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; import com.sleepycat.bind.tuple.IntegerBinding; import com.sleepycat.je.CheckpointConfig; import com.sleepycat.je.Cursor; import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseConfig; import com.sleepycat.je.DatabaseEntry; import com.sleepycat.je.DatabaseException; import com.sleepycat.je.DbInternal; import com.sleepycat.je.Environment; import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.OperationStatus; import com.sleepycat.je.Transaction; import com.sleepycat.je.config.EnvironmentParams; import com.sleepycat.je.dbi.DatabaseImpl; import com.sleepycat.je.dbi.EnvironmentImpl; import com.sleepycat.je.log.FileManager; import com.sleepycat.je.log.LogEntryHeader; import com.sleepycat.je.log.LogManager; import com.sleepycat.je.log.LogSource; import com.sleepycat.je.util.TestUtils; import com.sleepycat.je.utilint.DbLsn; /** * Test utilization counting of LNs. */ public class UtilizationTest extends TestCase { private static final String DB_NAME = "foo"; private static final String OP_NONE = "op-none"; private static final String OP_CHECKPOINT = "op-checkpoint"; private static final String OP_RECOVER = "op-recover"; //private static final String[] OPERATIONS = { OP_NONE, }; //* private static final String[] OPERATIONS = { OP_NONE, OP_CHECKPOINT, OP_RECOVER, OP_RECOVER }; //*/ /* * Set fetchObsoleteSize=true only for the second OP_RECOVER test. * We check that OP_RECOVER works with without fetching, but with fetching * we check that all LN sizes are counted. */ private static final boolean[] FETCH_OBSOLETE_SIZE = { false, false, false, true }; private static final CheckpointConfig forceConfig = new CheckpointConfig(); static { forceConfig.setForce(true); } private File envHome; private Environment env; private EnvironmentImpl envImpl; private Database db; private DatabaseImpl dbImpl; private boolean dups = false; private DatabaseEntry keyEntry = new DatabaseEntry(); private DatabaseEntry dataEntry = new DatabaseEntry(); private String operation; private long lastFileSeen; private boolean fetchObsoleteSize; private boolean truncateOrRemoveDone; public static Test suite() { TestSuite allTests = new TestSuite(); for (int i = 0; i < OPERATIONS.length; i += 1) { TestSuite suite = new TestSuite(UtilizationTest.class); Enumeration e = suite.tests(); while (e.hasMoreElements()) { UtilizationTest test = (UtilizationTest) e.nextElement(); test.init(OPERATIONS[i], FETCH_OBSOLETE_SIZE[i]); allTests.addTest(test); } } return allTests; } public UtilizationTest() { envHome = new File(System.getProperty(TestUtils.DEST_DIR)); } private void init(String operation, boolean fetchObsoleteSize) { this.operation = operation; this.fetchObsoleteSize = fetchObsoleteSize; } public void setUp() throws IOException, DatabaseException { TestUtils.removeLogFiles("Setup", envHome, false); TestUtils.removeFiles("Setup", envHome, FileManager.DEL_SUFFIX); } public void tearDown() throws IOException, DatabaseException { /* Set test name for reporting; cannot be done in the ctor or setUp. */ setName(operation + (fetchObsoleteSize ? "-fetch" : "") + ':' + getName()); try { if (env != null) { env.close(); } } catch (Throwable e) { System.out.println("tearDown: " + e); } try { //* TestUtils.removeLogFiles("tearDown", envHome, true); TestUtils.removeFiles("tearDown", envHome, FileManager.DEL_SUFFIX); //*/ } catch (Throwable e) { System.out.println("tearDown: " + e); } db = null; dbImpl = null; env = null; envImpl = null; envHome = null; keyEntry = null; dataEntry = null; } /** * Opens the environment and database. */ private void openEnv() throws DatabaseException { EnvironmentConfig config = TestUtils.initEnvConfig(); DbInternal.disableParameterValidation(config); config.setTransactional(true); config.setTxnNoSync(true); config.setAllowCreate(true); /* Do not run the daemons. */ config.setConfigParam (EnvironmentParams.ENV_RUN_CLEANER.getName(), "false"); config.setConfigParam (EnvironmentParams.ENV_RUN_EVICTOR.getName(), "false"); config.setConfigParam (EnvironmentParams.ENV_RUN_CHECKPOINTER.getName(), "false"); config.setConfigParam (EnvironmentParams.ENV_RUN_INCOMPRESSOR.getName(), "false"); /* Use a tiny log file size to write one LN per file. */ config.setConfigParam(EnvironmentParams.LOG_FILE_MAX.getName(), Integer.toString(64)); /* Don't use NIO direct buffers or we run out of memory. */ config.setConfigParam (EnvironmentParams.LOG_DIRECT_NIO.getName(), "false"); /* Obsolete LN size counting is optional per test. */ if (fetchObsoleteSize) { config.setConfigParam (EnvironmentParams.CLEANER_FETCH_OBSOLETE_SIZE.getName(), "true"); } env = new Environment(envHome, config); envImpl = DbInternal.envGetEnvironmentImpl(env); /* Speed up test that uses lots of very small files. */ envImpl.getFileManager().setSyncAtFileEnd(false); openDb(); } /** * Opens the database. */ private void openDb() throws DatabaseException { DatabaseConfig dbConfig = new DatabaseConfig(); dbConfig.setTransactional(true); dbConfig.setAllowCreate(true); dbConfig.setSortedDuplicates(dups); db = env.openDatabase(null, DB_NAME, dbConfig); dbImpl = DbInternal.dbGetDatabaseImpl(db); } /** * Closes the environment and database. */ private void closeEnv(boolean doCheckpoint) throws DatabaseException { /* * We pass expectAccurateDbUtilization as false when * truncateOrRemoveDone, because the database utilization info for that * database is now gone. */ VerifyUtils.verifyUtilization (envImpl, true, // expectAccurateObsoleteLNCount expectAccurateObsoleteLNSize(), !truncateOrRemoveDone); // expectAccurateDbUtilization if (db != null) { db.close(); db = null; } if (env != null) { envImpl.close(doCheckpoint); env = null; } } public void testReuseSlotAfterDelete() throws DatabaseException { openEnv(); /* Insert and delete without compress to create a knownDeleted slot. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, txn); long file1 = doDelete(0, txn); txn.commit(); /* Insert key 0 to reuse the knownDeleted slot. */ txn = env.beginTransaction(null, null); long file2 = doPut(0, txn); /* Delete and insert to reuse deleted slot in same txn. */ long file3 = doDelete(0, txn); long file4 = doPut(0, txn); txn.commit(); performRecoveryOperation(); expectObsolete(file0, true); expectObsolete(file1, true); expectObsolete(file2, true); expectObsolete(file3, true); expectObsolete(file4, false); closeEnv(true); } public void testReuseKnownDeletedSlot() throws DatabaseException { openEnv(); /* Insert key 0 and abort to create a knownDeleted slot. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, txn); txn.abort(); /* Insert key 0 to reuse the knownDeleted slot. */ txn = env.beginTransaction(null, null); long file1 = doPut(0, txn); txn.commit(); performRecoveryOperation(); /* Verify that file0 is still obsolete. */ expectObsolete(file0, true); expectObsolete(file1, false); closeEnv(true); } public void testReuseKnownDeletedSlotAbort() throws DatabaseException { openEnv(); /* Insert key 0 and abort to create a knownDeleted slot. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, txn); txn.abort(); /* Insert key 0 to reuse the knownDeleted slot, and abort. */ txn = env.beginTransaction(null, null); long file1 = doPut(0, txn); txn.abort(); performRecoveryOperation(); /* Verify that file0 is still obsolete. */ expectObsolete(file0, true); expectObsolete(file1, true); closeEnv(true); } public void testReuseKnownDeletedSlotDup() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Insert {0, 2} and abort to create a knownDeleted slot. */ txn = env.beginTransaction(null, null); long file3 = doPut(0, 2, txn); // 3rd LN long file4 = file3 + 1; // DupCountLN txn.abort(); /* Insert {0, 2} to reuse the knownDeleted slot. */ txn = env.beginTransaction(null, null); long file5 = doPut(0, 2, txn); // 4th LN long file6 = file5 + 1; // DupCountLN txn.commit(); performRecoveryOperation(); /* Verify that file3 is still obsolete. */ expectObsolete(file0, false); expectObsolete(file1, true); expectObsolete(file2, false); expectObsolete(file3, true); expectObsolete(file4, true); expectObsolete(file5, false); expectObsolete(file6, false); closeEnv(true); } public void testReuseKnownDeletedSlotDupAbort() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Insert {0, 2} and abort to create a knownDeleted slot. */ txn = env.beginTransaction(null, null); long file3 = doPut(0, 2, txn); // 3rd LN long file4 = file3 + 1; // DupCountLN txn.abort(); /* Insert {0, 2} to reuse the knownDeleted slot, then abort. */ txn = env.beginTransaction(null, null); long file5 = doPut(0, 2, txn); // 4th LN long file6 = file5 + 1; // DupCountLN txn.abort(); performRecoveryOperation(); /* Verify that file3 is still obsolete. */ expectObsolete(file0, false); expectObsolete(file1, false); expectObsolete(file2, false); expectObsolete(file3, true); expectObsolete(file4, true); expectObsolete(file5, true); expectObsolete(file6, true); closeEnv(true); } public void testInsert() throws DatabaseException { openEnv(); /* Insert key 0. */ long file0 = doPut(0, true); performRecoveryOperation(); /* Expect that LN is not obsolete. */ FileSummary fileSummary = getFileSummary(file0); assertEquals(1, fileSummary.totalLNCount); assertEquals(0, fileSummary.obsoleteLNCount); DbFileSummary dbFileSummary = getDbFileSummary(file0); assertEquals(1, dbFileSummary.totalLNCount); assertEquals(0, dbFileSummary.obsoleteLNCount); closeEnv(true); } public void testInsertAbort() throws DatabaseException { openEnv(); /* Insert key 0. */ long file0 = doPut(0, false); performRecoveryOperation(); /* Expect that LN is obsolete. */ FileSummary fileSummary = getFileSummary(file0); assertEquals(1, fileSummary.totalLNCount); assertEquals(1, fileSummary.obsoleteLNCount); DbFileSummary dbFileSummary = getDbFileSummary(file0); assertEquals(1, dbFileSummary.totalLNCount); assertEquals(1, dbFileSummary.obsoleteLNCount); closeEnv(true); } public void testInsertDup() throws DatabaseException { dups = true; openEnv(); /* Insert key 0 and a dup. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); long file3 = doPut(0, 1, txn); txn.commit(); performRecoveryOperation(); /* * The dup tree is created on 2nd insert. In between the two * DupCountLNs are two INs. */ long file1 = file0 + 1; // DupCountLN (provisional) long file2 = file1 + 3; // DupCountLN (non-provisional) assertEquals(file3, file2 + 1); // new LN expectObsolete(file0, false); // 1st LN expectObsolete(file1, true); // 1st DupCountLN expectObsolete(file2, false); // 2nd DupCountLN expectObsolete(file3, false); // 2nd LN closeEnv(true); } public void testInsertDupAbort() throws DatabaseException { dups = true; openEnv(); /* Insert key 0 and a dup. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); long file3 = doPut(0, 1, txn); txn.abort(); performRecoveryOperation(); /* * The dup tree is created on 2nd insert. In between the two * DupCountLNs are two INs. */ long file1 = file0 + 1; // DupCountLN (provisional) long file2 = file1 + 3; // DupCountLN (non-provisional) assertEquals(file3, file2 + 1); // new LN expectObsolete(file0, true); // 1st LN expectObsolete(file1, false); // 1st DupCountLN expectObsolete(file2, true); // 2nd DupCountLN expectObsolete(file3, true); // 2nd LN closeEnv(true); } public void testUpdate() throws DatabaseException { openEnv(); /* Insert key 0 and checkpoint. */ long file0 = doPut(0, true); env.checkpoint(forceConfig); /* Update key 0. */ long file1 = doPut(0, true); performRecoveryOperation(); expectObsolete(file0, true); expectObsolete(file1, false); closeEnv(true); } public void testUpdateAbort() throws DatabaseException { openEnv(); /* Insert key 0 and checkpoint. */ long file0 = doPut(0, true); env.checkpoint(forceConfig); /* Update key 0 and abort. */ long file1 = doPut(0, false); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, true); closeEnv(true); } public void testUpdateDup() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Update {0, 0}. */ txn = env.beginTransaction(null, null); long file3 = doUpdate(0, 0, txn); // 3rd LN txn.commit(); performRecoveryOperation(); expectObsolete(file0, true); expectObsolete(file1, false); expectObsolete(file2, false); expectObsolete(file3, false); closeEnv(true); } public void testUpdateDupAbort() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Update {0, 0}. */ txn = env.beginTransaction(null, null); long file3 = doUpdate(0, 0, txn); // 3rd LN txn.abort(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, false); expectObsolete(file2, false); expectObsolete(file3, true); closeEnv(true); } public void testDelete() throws DatabaseException { openEnv(); /* Insert key 0 and checkpoint. */ long file0 = doPut(0, true); env.checkpoint(forceConfig); /* Delete key 0. */ long file1 = doDelete(0, true); performRecoveryOperation(); expectObsolete(file0, true); expectObsolete(file1, true); closeEnv(true); } public void testDeleteAbort() throws DatabaseException { openEnv(); /* Insert key 0 and checkpoint. */ long file0 = doPut(0, true); env.checkpoint(forceConfig); /* Delete key 0 and abort. */ long file1 = doDelete(0, false); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, true); closeEnv(true); } public void testDeleteDup() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Delete {0, 0} and abort. */ txn = env.beginTransaction(null, null); long file3 = doDelete(0, 0, txn); // 3rd LN long file4 = file3 + 1; // DupCountLN txn.commit(); performRecoveryOperation(); expectObsolete(file0, true); expectObsolete(file1, true); expectObsolete(file2, false); expectObsolete(file3, true); expectObsolete(file4, false); closeEnv(true); } public void testDeleteDupAbort() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Delete {0, 0} and abort. */ txn = env.beginTransaction(null, null); long file3 = doDelete(0, 0, txn); // 3rd LN long file4 = file3 + 1; // DupCountLN txn.abort(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, false); expectObsolete(file2, false); expectObsolete(file3, true); expectObsolete(file4, true); closeEnv(true); } public void testInsertUpdate() throws DatabaseException { openEnv(); /* Insert and update key 0. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, txn); long file1 = doPut(0, txn); txn.commit(); performRecoveryOperation(); expectObsolete(file0, true); expectObsolete(file1, false); closeEnv(true); } public void testInsertUpdateAbort() throws DatabaseException { openEnv(); /* Insert and update key 0. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, txn); long file1 = doPut(0, txn); txn.abort(); performRecoveryOperation(); expectObsolete(file0, true); expectObsolete(file1, true); closeEnv(true); } public void testInsertUpdateDup() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Insert and update {0, 2}. */ txn = env.beginTransaction(null, null); long file3 = doPut(0, 2, txn); // 3rd LN long file4 = file3 + 1; // DupCountLN long file5 = doUpdate(0, 2, txn); // 4rd LN txn.commit(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, true); expectObsolete(file2, false); expectObsolete(file3, true); expectObsolete(file4, false); expectObsolete(file5, false); closeEnv(true); } public void testInsertUpdateDupAbort() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Insert and update {0, 2}. */ txn = env.beginTransaction(null, null); long file3 = doPut(0, 2, txn); // 3rd LN long file4 = file3 + 1; // DupCountLN long file5 = doUpdate(0, 2, txn); // 4rd LN txn.abort(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, false); expectObsolete(file2, false); expectObsolete(file3, true); expectObsolete(file4, true); expectObsolete(file5, true); closeEnv(true); } public void testInsertDelete() throws DatabaseException { openEnv(); /* Insert and update key 0. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, txn); long file1 = doDelete(0, txn); txn.commit(); performRecoveryOperation(); expectObsolete(file0, true); expectObsolete(file1, true); closeEnv(true); } public void testInsertDeleteAbort() throws DatabaseException { openEnv(); /* Insert and update key 0. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, txn); long file1 = doDelete(0, txn); txn.abort(); performRecoveryOperation(); expectObsolete(file0, true); expectObsolete(file1, true); closeEnv(true); } public void testInsertDeleteDup() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Insert and delete {0, 2}. */ txn = env.beginTransaction(null, null); long file3 = doPut(0, 2, txn); // 3rd LN long file4 = file3 + 1; // DupCountLN long file5 = doDelete(0, 2, txn); // 4rd LN long file6 = file5 + 1; // DupCountLN txn.commit(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, true); expectObsolete(file2, false); expectObsolete(file3, true); expectObsolete(file4, true); expectObsolete(file5, true); expectObsolete(file6, false); closeEnv(true); } public void testInsertDeleteDupAbort() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Insert and delete {0, 2} and abort. */ txn = env.beginTransaction(null, null); long file3 = doPut(0, 2, txn); // 3rd LN long file4 = file3 + 1; // DupCountLN long file5 = doDelete(0, 2, txn); // 4rd LN long file6 = file5 + 1; // DupCountLN txn.abort(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, false); expectObsolete(file2, false); expectObsolete(file3, true); expectObsolete(file4, true); expectObsolete(file5, true); expectObsolete(file6, true); closeEnv(true); } public void testUpdateUpdate() throws DatabaseException { openEnv(); /* Insert key 0 and checkpoint. */ long file0 = doPut(0, true); env.checkpoint(forceConfig); /* Update key 0 twice. */ Transaction txn = env.beginTransaction(null, null); long file1 = doPut(0, txn); long file2 = doPut(0, txn); txn.commit(); performRecoveryOperation(); expectObsolete(file0, true); expectObsolete(file1, true); expectObsolete(file2, false); closeEnv(true); } public void testUpdateUpdateAbort() throws DatabaseException { openEnv(); /* Insert key 0 and checkpoint. */ long file0 = doPut(0, true); env.checkpoint(forceConfig); /* Update key 0 twice and abort. */ Transaction txn = env.beginTransaction(null, null); long file1 = doPut(0, txn); long file2 = doPut(0, txn); txn.abort(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, true); expectObsolete(file2, true); closeEnv(true); } public void testUpdateUpdateDup() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Update {0, 1} twice. */ txn = env.beginTransaction(null, null); long file3 = doUpdate(0, 1, txn); // 3rd LN long file4 = doUpdate(0, 1, txn); // 4rd LN txn.commit(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, false); expectObsolete(file2, true); expectObsolete(file3, true); expectObsolete(file4, false); closeEnv(true); } public void testUpdateUpdateDupAbort() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Update {0, 1} twice and abort. */ txn = env.beginTransaction(null, null); long file3 = doUpdate(0, 1, txn); // 3rd LN long file4 = doUpdate(0, 1, txn); // 4rd LN txn.abort(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, false); expectObsolete(file2, false); expectObsolete(file3, true); expectObsolete(file4, true); closeEnv(true); } public void testUpdateDelete() throws DatabaseException { openEnv(); /* Insert key 0 and checkpoint. */ long file0 = doPut(0, true); env.checkpoint(forceConfig); /* Update and delete key 0. */ Transaction txn = env.beginTransaction(null, null); long file1 = doPut(0, txn); long file2 = doDelete(0, txn); txn.commit(); performRecoveryOperation(); expectObsolete(file0, true); expectObsolete(file1, true); expectObsolete(file2, true); closeEnv(true); } public void testUpdateDeleteAbort() throws DatabaseException { openEnv(); /* Insert key 0 and checkpoint. */ long file0 = doPut(0, true); env.checkpoint(forceConfig); /* Update and delete key 0 and abort. */ Transaction txn = env.beginTransaction(null, null); long file1 = doPut(0, txn); long file2 = doDelete(0, txn); txn.abort(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, true); expectObsolete(file2, true); closeEnv(true); } public void testUpdateDeleteDup() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Update and delete {0, 1}. */ txn = env.beginTransaction(null, null); long file3 = doUpdate(0, 1, txn); // 3rd LN long file4 = doDelete(0, 1, txn); // 4rd LN long file5 = file4 + 1; // DupCountLN txn.commit(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, true); expectObsolete(file2, true); expectObsolete(file3, true); expectObsolete(file4, true); expectObsolete(file5, false); closeEnv(true); } public void testUpdateDeleteDupAbort() throws DatabaseException { dups = true; openEnv(); /* Insert two key 0 dups and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, 0, txn); // 1st LN long file2 = doPut(0, 1, txn); // 2nd LN long file1 = file2 - 1; // DupCountLN txn.commit(); env.checkpoint(forceConfig); /* Update and delete {0, 1} and abort. */ txn = env.beginTransaction(null, null); long file3 = doUpdate(0, 1, txn); // 3rd LN long file4 = doDelete(0, 1, txn); // 4rd LN long file5 = file4 + 1; // DupCountLN txn.abort(); performRecoveryOperation(); expectObsolete(file0, false); expectObsolete(file1, false); expectObsolete(file2, false); expectObsolete(file3, true); expectObsolete(file4, true); expectObsolete(file5, true); closeEnv(true); } public void testTruncate() throws DatabaseException { truncateOrRemove(true, true); } public void testTruncateAbort() throws DatabaseException { truncateOrRemove(true, false); } public void testRemove() throws DatabaseException { truncateOrRemove(false, true); } public void testRemoveAbort() throws DatabaseException { truncateOrRemove(false, false); } /** */ private void truncateOrRemove(boolean truncate, boolean commit) throws DatabaseException { openEnv(); /* Insert 3 keys and checkpoint. */ Transaction txn = env.beginTransaction(null, null); long file0 = doPut(0, txn); long file1 = doPut(1, txn); long file2 = doPut(2, txn); txn.commit(); env.checkpoint(forceConfig); /* Truncate. */ txn = env.beginTransaction(null, null); if (truncate) { db.close(); db = null; long count = env.truncateDatabase(txn, DB_NAME, true /* returnCount */); assertEquals(3, count); } else { db.close(); db = null; env.removeDatabase(txn, DB_NAME); } if (commit) { txn.commit(); } else { txn.abort(); } truncateOrRemoveDone = true; performRecoveryOperation(); /* * Do not check DbFileSummary when we truncate/remove, since the old * DatabaseImpl is gone. */ expectObsolete(file0, commit, !commit /*checkDbFileSummary*/); expectObsolete(file1, commit, !commit /*checkDbFileSummary*/); expectObsolete(file2, commit, !commit /*checkDbFileSummary*/); closeEnv(true); } /* * The xxxForceTreeWalk tests set the DatabaseImpl * forceTreeWalkForTruncateAndRemove field to true, which will force a walk * of the tree to count utilization during truncate/remove, rather than * using the per-database info. This is used to test the "old technique" * for counting utilization, which is now used only if the database was * created prior to log version 6. */ public void testTruncateForceTreeWalk() throws Exception { DatabaseImpl.forceTreeWalkForTruncateAndRemove = true; try { testTruncate(); } finally { DatabaseImpl.forceTreeWalkForTruncateAndRemove = false; } } public void testTruncateAbortForceTreeWalk() throws Exception { DatabaseImpl.forceTreeWalkForTruncateAndRemove = true; try { testTruncateAbort(); } finally { DatabaseImpl.forceTreeWalkForTruncateAndRemove = false; } } public void testRemoveForceTreeWalk() throws Exception { DatabaseImpl.forceTreeWalkForTruncateAndRemove = true; try { testRemove(); } finally { DatabaseImpl.forceTreeWalkForTruncateAndRemove = false; } } public void testRemoveAbortForceTreeWalk() throws Exception { DatabaseImpl.forceTreeWalkForTruncateAndRemove = true; try { testRemoveAbort(); } finally { DatabaseImpl.forceTreeWalkForTruncateAndRemove = false; } } private void expectObsolete(long file, boolean obsolete) throws DatabaseException { expectObsolete(file, obsolete, true /*checkDbFileSummary*/); } private void expectObsolete(long file, boolean obsolete, boolean checkDbFileSummary) throws DatabaseException { FileSummary fileSummary = getFileSummary(file); assertEquals("totalLNCount", 1, fileSummary.totalLNCount); assertEquals("obsoleteLNCount", obsolete ? 1 : 0, fileSummary.obsoleteLNCount); DbFileSummary dbFileSummary = getDbFileSummary(file); if (checkDbFileSummary) { assertEquals("db totalLNCount", 1, dbFileSummary.totalLNCount); assertEquals("db obsoleteLNCount", obsolete ? 1 : 0, dbFileSummary.obsoleteLNCount); } if (obsolete) { if (expectAccurateObsoleteLNSize()) { assertTrue(fileSummary.obsoleteLNSize > 0); assertEquals(1, fileSummary.obsoleteLNSizeCounted); if (checkDbFileSummary) { assertTrue(dbFileSummary.obsoleteLNSize > 0); assertEquals(1, dbFileSummary.obsoleteLNSizeCounted); } } /* If we counted the size, make sure it is the actual LN size. */ if (fileSummary.obsoleteLNSize > 0) { assertEquals(getLNSize(file), fileSummary.obsoleteLNSize); } if (checkDbFileSummary) { if (dbFileSummary.obsoleteLNSize > 0) { assertEquals(getLNSize(file), dbFileSummary.obsoleteLNSize); } assertEquals(fileSummary.obsoleteLNSize > 0, dbFileSummary.obsoleteLNSize > 0); } } else { assertEquals(0, fileSummary.obsoleteLNSize); assertEquals(0, fileSummary.obsoleteLNSizeCounted); if (checkDbFileSummary) { assertEquals(0, dbFileSummary.obsoleteLNSize); assertEquals(0, dbFileSummary.obsoleteLNSizeCounted); } } } /** * If an LN is obsolete, expect the size to be counted unless we ran * recovery and we did NOT configure fetchObsoleteSize=true. In that * case, the size may or may not be counted depending on how the redo * or undo was processed during reocvery. */ private boolean expectAccurateObsoleteLNSize() { return fetchObsoleteSize || !OP_RECOVER.equals(operation); } private long doPut(int key, boolean commit) throws DatabaseException { Transaction txn = env.beginTransaction(null, null); long file = doPut(key, txn); if (commit) { txn.commit(); } else { txn.abort(); } return file; } private long doPut(int key, Transaction txn) throws DatabaseException { return doPut(key, key, txn); } private long doPut(int key, int data, Transaction txn) throws DatabaseException { Cursor cursor = db.openCursor(txn, null); IntegerBinding.intToEntry(key, keyEntry); IntegerBinding.intToEntry(data, dataEntry); cursor.put(keyEntry, dataEntry); long file = getFile(cursor); cursor.close(); return file; } private long doUpdate(int key, int data, Transaction txn) throws DatabaseException { Cursor cursor = db.openCursor(txn, null); IntegerBinding.intToEntry(key, keyEntry); IntegerBinding.intToEntry(data, dataEntry); assertEquals(OperationStatus.SUCCESS, cursor.getSearchBoth(keyEntry, dataEntry, null)); cursor.putCurrent(dataEntry); long file = getFile(cursor); cursor.close(); return file; } private long doDelete(int key, boolean commit) throws DatabaseException { Transaction txn = env.beginTransaction(null, null); long file = doDelete(key, txn); if (commit) { txn.commit(); } else { txn.abort(); } return file; } private long doDelete(int key, Transaction txn) throws DatabaseException { Cursor cursor = db.openCursor(txn, null); IntegerBinding.intToEntry(key, keyEntry); assertEquals(OperationStatus.SUCCESS, cursor.getSearchKey(keyEntry, dataEntry, null)); cursor.delete(); long file = getFile(cursor); cursor.close(); return file; } private long doDelete(int key, int data, Transaction txn) throws DatabaseException { Cursor cursor = db.openCursor(txn, null); IntegerBinding.intToEntry(key, keyEntry); IntegerBinding.intToEntry(data, dataEntry); assertEquals(OperationStatus.SUCCESS, cursor.getSearchBoth(keyEntry, dataEntry, null)); cursor.delete(); long file = getFile(cursor); cursor.close(); return file; } /** * Checkpoint, recover, or do neither, depending on the configured * operation for this test. Always compress to count deleted LNs. */ private void performRecoveryOperation() throws DatabaseException { if (OP_NONE.equals(operation)) { /* Compress to count deleted LNs. */ env.compress(); } else if (OP_CHECKPOINT.equals(operation)) { /* Compress before checkpointing to count deleted LNs. */ env.compress(); env.checkpoint(forceConfig); } else if (OP_RECOVER.equals(operation)) { closeEnv(false); openEnv(); /* Compress after recovery to count deleted LNs. */ env.compress(); } else { assert false : operation; } } /** * Gets the file of the LSN at the cursor position, using internal methods. * Also check that the file number is greater than the last file returned, * to ensure that we're filling a file every time we write. */ private long getFile(Cursor cursor) throws DatabaseException { long file = CleanerTestUtils.getLogFile(this, cursor); assert file > lastFileSeen; lastFileSeen = file; return file; } /** * Returns the utilization summary for a given log file. */ private FileSummary getFileSummary(long file) throws DatabaseException { return (FileSummary) envImpl.getUtilizationProfile() .getFileSummaryMap(true) .get(new Long(file)); } /** * Returns the per-database utilization summary for a given log file. */ private DbFileSummary getDbFileSummary(long file) { return dbImpl.getDbFileSummary (new Long(file), false /*willModify*/); } /** * Peek into the file to get the total size of the first entry past the * file header, which is known to be the LN log entry. */ private int getLNSize(long file) throws DatabaseException { try { long offset = FileManager.firstLogEntryOffset(); long lsn = DbLsn.makeLsn(file, offset); LogManager lm = envImpl.getLogManager(); LogSource src = lm.getLogSource(lsn); ByteBuffer buf = src.getBytes(offset); LogEntryHeader header = new LogEntryHeader(null, // envImpl, only needed for buf, // error reporting false); // anticipateChecksumError int size = header.getItemSize(); src.release(); return size + header.getSize(); } catch (IOException e) { throw new DatabaseException(e); } } }