/* * Copyright 2007 - 2018 ETH Zuerich, CISD and SIS. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package ch.systemsx.cisd.hdf5.h5ar; import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertFalse; import static org.testng.AssertJUnit.assertNotNull; import static org.testng.AssertJUnit.assertNull; import static org.testng.AssertJUnit.assertTrue; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintStream; import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; import org.testng.annotations.AfterTest; import org.testng.annotations.BeforeSuite; import org.testng.annotations.Test; import ch.systemsx.cisd.base.unix.FileLinkType; import ch.systemsx.cisd.base.unix.Unix; import ch.systemsx.cisd.base.unix.Unix.Stat; import ch.systemsx.cisd.base.utilities.OSUtilities; import ch.systemsx.cisd.hdf5.HDF5Factory; import ch.systemsx.cisd.hdf5.IHDF5Writer; import ch.systemsx.cisd.hdf5.IHDF5WriterConfigurator.FileFormatVersionBounds; /** * Tests for {@link HDF5Archiver}. * * @author Bernd Rinn */ public class HDF5ArchiverTest { private static final File rootDirectory = new File("targets", "unit-test-wd"); private static final File workingDirectory = new File(rootDirectory, "hdf5-archivertest-wd"); private static final File srcDirectory = new File("sourceTest/java/ch/systemsx/cisd/hdf5/h5ar"); @BeforeSuite public void init() { workingDirectory.mkdirs(); assertTrue(workingDirectory.isDirectory()); workingDirectory.deleteOnExit(); rootDirectory.deleteOnExit(); } @AfterTest public void cleanup() { deleteAll(workingDirectory); } private void deleteAll(File path) { if (path.isDirectory()) { for (File sub : path.listFiles()) { deleteAll(sub); } } path.delete(); } @Override protected void finalize() throws Throwable { // Delete the working directory if (workingDirectory.exists() && workingDirectory.canWrite()) { workingDirectory.delete(); } // Delete root directory if (rootDirectory.exists() && rootDirectory.canWrite()) { rootDirectory.delete(); } super.finalize(); } @Test public void testGetPermissions() { assertEquals("----------", Utils.permissionsToString(0, false, false)); assertEquals("-rw-rw-rw-", Utils.permissionsToString(0666, false, false)); assertEquals("-r--r--r--", Utils.permissionsToString(0444, false, false)); assertEquals("-rwx------", Utils.permissionsToString(0700, false, false)); assertEquals("-rwsr-xr-x", Utils.permissionsToString(04755, false, false)); assertEquals("-rwSr-xr-x", Utils.permissionsToString(04655, false, false)); assertEquals("-rwxr-sr-x", Utils.permissionsToString(02755, false, false)); assertEquals("-rwxr-Sr-x", Utils.permissionsToString(02745, false, false)); assertEquals("-rwxr-xr-t", Utils.permissionsToString(01755, false, false)); assertEquals("-rwxr-xr-T", Utils.permissionsToString(01754, false, false)); assertEquals("d---------", Utils.permissionsToString(0, true, false)); assertEquals("drwxr-xr-x", Utils.permissionsToString(0755, true, false)); } @Test(groups = { "requires_unix" }) public void testDescribeLink() { final String rootGroupName = OSUtilities.isMacOS() ? "wheel" : "root"; final IdCache idCache = new IdCache(); assertEquals("dir/link_name", new ArchiveEntry("dir", "dir/link_name", new LinkRecord(null, null, null, -1, -1, -1, -1, (short) -1, 0), idCache).describeLink(false, false)); assertEquals(" 100\t00000000\tdir/link_name", new ArchiveEntry("dir", "dir/link_name", new LinkRecord(null, null, FileLinkType.REGULAR_FILE, 100, -1, -1, -1, (short) -1, 0), idCache).describeLink(true, false)); assertEquals("-rwxr-xr-x\troot\t" + rootGroupName + "\t 111\t2000-01-01 00:00:00\t00000000\tdir/link_name", new ArchiveEntry( "dir", "dir/link_name", new LinkRecord(null, null, FileLinkType.REGULAR_FILE, 111L, 946681200491L / 1000L, 0, 0, (short) 0755, 0), idCache).describeLink(true, false)); assertEquals("d---------\troot\t" + rootGroupName + "\t DIR\t2000-01-01 00:00:00\t \tdir/link_name", new ArchiveEntry( "dir", "dir/link_name", new LinkRecord(null, null, FileLinkType.DIRECTORY, 111L, 946681200491L / 1000L, 0, 0, (short) 0, 0), idCache).describeLink(true, false)); assertEquals("755\troot\troot\t 111\t2000-01-01 00:00:00\t" + Utils.crc32ToString(200) + "\tdir/link_name", new ArchiveEntry("dir", "dir/link_name", new LinkRecord(null, null, FileLinkType.REGULAR_FILE, 111L, 946681200491L / 1000L, 0, 0, (short) 0755, 200), idCache).describeLink(true, true)); assertEquals("0\troot\troot\t DIR\t2000-01-01 00:00:00\t \tdir/link_name", new ArchiveEntry("dir", "dir/link_name", new LinkRecord("link_name2", null, FileLinkType.DIRECTORY, 111L, 946681200491L / 1000L, 0, 0, (short) 0, 0), idCache).describeLink(true, true)); assertEquals(" 111\t2000-01-01 00:00:00\t00000000\tdir/link_name", new ArchiveEntry( "dir", "dir/link_name", new LinkRecord("link_name", null, FileLinkType.REGULAR_FILE, 111L, 946681200491L / 1000L, -1, 0, (short) 0755, 0), idCache).describeLink(true, false)); assertEquals(" 111\t00000000\tdir/link_name", new ArchiveEntry("dir", "dir/link_name", new LinkRecord("link_name2", null, FileLinkType.REGULAR_FILE, 111L, -1L, -1, 0, (short) 0755, 0), idCache).describeLink(true, false)); } @Test(groups = { "requires_unix" }) public void testIdCache() { if (Unix.isOperational() == false) { return; } final int uid = Unix.getUid(); final String uname = Unix.tryGetUserNameForUid(uid); final IdCache idCache = new IdCache(); assertEquals("-17", idCache.getUser(-17, true)); assertEquals("root", idCache.getUser(0, false)); assertEquals(uname, idCache.getUser(uid, false)); Integer invalidUid = getInvalidUid(); if (invalidUid != null) { assertEquals(Integer.toString(invalidUid), idCache.getUser(invalidUid, false)); } } private Integer getInvalidUid() { int invalidUid; for (invalidUid = 60000; invalidUid < 65535 && Unix.tryGetUserByUid(invalidUid) != null; ++invalidUid) { } return Unix.tryGetUserByUid(invalidUid) == null ? invalidUid : null; } private void writeToArchive(final IHDF5Archiver a, final String name, final String content) { final byte[] bytes = content.getBytes(); a.archiveFile(NewArchiveEntry.file("/test", name).lastModified(1000000L).uid(100).gid(100), new ByteArrayInputStream(bytes)); } @Test public void testWriteByteArrayToArchive() { final File file = new File(workingDirectory, "writeByteArrayToArchive.h5ar"); file.delete(); file.deleteOnExit(); final IHDF5Archiver a = HDF5ArchiverFactory.open(file, false, FileFormatVersionBounds.LATEST_LATEST, null); writeToArchive(a, "hello.txt", "Hello World\n"); writeToArchive(a, "hello2.txt", "Yet another Hello World\n"); a.close(); final IHDF5ArchiveReader aro = HDF5ArchiverFactory.openForReading(file); final String content1 = new String(aro.extractFileAsByteArray("/test/hello.txt")); assertEquals("Hello World\n", content1); final String content2 = new String(aro.extractFileAsByteArray("/test/hello2.txt")); assertEquals("Yet another Hello World\n", content2); final List list = aro.list("/", ListParameters.build().nonRecursive().noReadLinkTarget().get()); assertEquals(1, list.size()); assertEquals("755\t100\t100\t DIR\t1970-01-12 14:46:40\t \t/test", list.get(0) .describeLink(true, true)); final List list2 = aro.list("/test", ListParameters.build().testArchive().suppressDirectoryEntries() .get()); assertEquals(2, list2.size()); assertEquals( "755\t100\t100\t 12\t1970-01-12 14:46:40\tb095e5e3\t/test/hello.txt\tOK", list2.get(0).describeLink(true, true)); assertEquals( "755\t100\t100\t 24\t1970-01-12 14:46:40\tee5f3107\t/test/hello2.txt\tOK", list2.get(1).describeLink(true, true)); aro.close(); assertTrue(HDF5Factory.hasMDCImage(file)); } @Test public void testWriteFileAsOutputStream() throws Exception { final File file = new File(workingDirectory, "writeFileAsOutputStream.h5ar"); file.delete(); file.deleteOnExit(); final IHDF5Archiver a = HDF5ArchiverFactory.open(file); final PrintStream ps = new PrintStream(a.archiveFileAsOutputStream(NewArchiveEntry.file("test1") .chunkSize(128))); ps.printf("Some %s stuff: %d\n", "more", 17); // Note: we don't close the PrintStream or the underlying OutputStream explicitly. // The flushables take care of things getting written correctly anyway. a.close(); final IHDF5ArchiveReader ar = HDF5ArchiverFactory.openForReading(file); final List entries = ar.list("", ListParameters.build().testArchive().get()); assertEquals(1, entries.size()); assertEquals("test1", entries.get(0).getName()); assertTrue(entries.get(0).isOK()); final BufferedReader r = new BufferedReader(new InputStreamReader(ar.extractFileAsInputStream("test1"))); assertEquals("Some more stuff: 17", r.readLine()); assertNull(r.readLine()); ar.close(); } private File createTestDirectory() throws IOException { return createTestDirectory(null, System.currentTimeMillis()); } private File createTestDirectory(String prefixOrNull, long time) throws IOException { final File prefixDir = (prefixOrNull != null) ? new File(workingDirectory, prefixOrNull) : workingDirectory; prefixDir.delete(); prefixDir.deleteOnExit(); final File dir = new File(prefixDir, "test"); dir.delete(); dir.deleteOnExit(); dir.mkdirs(); final File f1 = new File(dir, "file_test1.txt"); f1.delete(); f1.deleteOnExit(); FileUtils.writeLines(f1, Arrays.asList("Line 1", "Line 2", "Line 3")); f1.setLastModified(time); final File dir2 = new File(dir, "dir_somedir"); dir2.delete(); dir2.mkdir(); dir2.deleteOnExit(); final File f2 = new File(dir2, "file_test2.txt"); f2.delete(); f2.deleteOnExit(); FileUtils.writeLines(f2, Arrays.asList("A", "B", "C")); f2.setLastModified(time); final File dir3 = new File(dir, "dir_someotherdir"); dir3.delete(); dir3.mkdir(); dir3.deleteOnExit(); if (Unix.isOperational()) { final File l1 = new File(dir2, "link_todir3"); l1.delete(); l1.deleteOnExit(); final String linktarget = "../" + dir3.getName(); final String linkname = l1.getAbsolutePath(); Unix.createSymbolicLink(linktarget, linkname); final long linktime = time / Utils.MILLIS_PER_SECOND + 24 * 3600L; Unix.setLinkTimestamps(linkname, linktime, linktime); } dir2.setLastModified(time); dir3.setLastModified(time); dir.setLastModified(time); return dir; } @Test public void testCreateVerifyRoundtripOK() throws IOException { final File dir = createTestDirectory(); final File h5arfile = new File(workingDirectory, "testRoundtrip.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); HDF5ArchiverFactory.open(h5arfile).archiveFromFilesystem(dir).close(); final IHDF5ArchiveReader ar = HDF5ArchiverFactory.openForReading(h5arfile); assertTrue(ar.test().isEmpty()); assertTrue(ar.verifyAgainstFilesystem(dir).isEmpty()); ar.close(); } @Test public void testCreateVerifyContentArtificialRootRoundtripOK() throws IOException { final File dir = createTestDirectory(); final File h5arfile = new File(workingDirectory, "testRoundtripContentArtificialRoot.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); HDF5ArchiverFactory.open(h5arfile).archiveFromFilesystemBelowDirectory("ttt", dir).close(); final IHDF5ArchiveReader ar = HDF5ArchiverFactory.openForReading(h5arfile); assertTrue(ar.test().isEmpty()); assertTrue(ar.verifyAgainstFilesystem("", dir, "ttt").isEmpty()); ar.close(); } @Test public void testRoundtrip() throws IOException { final long now = System.currentTimeMillis(); final long dirLastChanged = now - 1000L * 3600L * 24 * 5; final File dir = createTestDirectory("original", dirLastChanged); final long dirLastChangedSeconds = dirLastChanged / 1000L; final File h5arfile = new File(workingDirectory, "testRoundtrip.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); final AtomicInteger entryCount = new AtomicInteger(0); HDF5ArchiverFactory.open(h5arfile) .archiveFromFilesystemBelowDirectory("/", dir, new IArchiveEntryVisitor() { @Override public void visit(ArchiveEntry entry) { entryCount.incrementAndGet(); final File f = new File(dir, entry.getPath()); assertTrue(entry.getPath(), f.exists()); assertTrue(entry.isOK()); if (entry.isSymLink() == false) { assertEquals(dirLastChangedSeconds, entry.getLastModified()); assertEquals(entry.getPath(), f.isDirectory(), entry.isDirectory()); assertEquals(entry.getPath(), f.isFile(), entry.isRegularFile()); if (entry.isRegularFile()) { assertEquals(entry.getPath(), f.length(), entry.getSize()); } } } }).close(); assertEquals(6, entryCount.intValue()); final IHDF5ArchiveReader ar = HDF5ArchiverFactory.openForReading(h5arfile); entryCount.set(0); ar.list("/", new IArchiveEntryVisitor() { @Override public void visit(ArchiveEntry entry) { entryCount.incrementAndGet(); final File f = new File(dir, entry.getPath()); assertTrue(entry.getPath(), f.exists()); assertTrue(entry.isOK()); if (entry.isSymLink() == false) { assertEquals(dirLastChangedSeconds, entry.getLastModified()); assertEquals(entry.getPath(), f.isDirectory(), entry.isDirectory()); assertEquals(entry.getPath(), f.isFile(), entry.isRegularFile()); if (entry.isRegularFile()) { assertEquals(entry.getPath(), f.length(), entry.getSize()); } } } }); assertEquals(5, entryCount.intValue()); assertTrue(ar.verifyAgainstFilesystem(dir).isEmpty()); final File extracted = new File(dir.getParentFile().getParentFile(), "extracted"); deleteAll(extracted); entryCount.set(0); ar.extractToFilesystem(extracted, "/", new IArchiveEntryVisitor() { @Override public void visit(ArchiveEntry entry) { entryCount.incrementAndGet(); final File f = new File(dir, entry.getPath()); assertTrue(entry.getPath(), f.exists()); assertTrue(entry.isOK()); if (entry.isSymLink() == false) { assertEquals(dirLastChangedSeconds, entry.getLastModified()); assertEquals(entry.getPath(), f.isDirectory(), entry.isDirectory()); assertEquals(entry.getPath(), f.isFile(), entry.isRegularFile()); if (entry.isRegularFile()) { assertEquals(entry.getPath(), f.length(), entry.getSize()); } } } }); assertEquals(5, entryCount.get()); for (ArchiveEntry s : ar.verifyAgainstFilesystem(extracted, VerifyParameters.build().verifyAttributes().get())) { System.out.println(s.getStatus(true)); } assertTrue(ar.verifyAgainstFilesystem(extracted, VerifyParameters.build().verifyAttributes().get()).isEmpty()); entryCount.set(0); checkDirectoryEntries(dir, extracted, entryCount); assertEquals(5, entryCount.intValue()); final File partiallyExtracted = new File(dir.getParentFile().getParentFile(), "partiallyExtracted"); deleteAll(partiallyExtracted); entryCount.set(0); final String[] pathsInDirSomedir = new String[] { "/dir_somedir", "/dir_somedir/file_test2.txt", "/dir_somedir/link_todir3" }; ar.extractToFilesystemBelowDirectory(partiallyExtracted, "/dir_somedir", new IArchiveEntryVisitor() { @Override public void visit(ArchiveEntry entry) { int idx = entryCount.getAndIncrement(); assertEquals(pathsInDirSomedir[idx], entry.getPath()); } }); assertEquals(3, entryCount.get()); ar.close(); } private void checkDirectoryEntries(final File dir, final File extracted, final AtomicInteger entryCount) { for (File f : extracted.listFiles()) { entryCount.incrementAndGet(); final String relativePath = f.getAbsolutePath().substring(extracted.getAbsolutePath().length() + 1); final File orig = new File(dir, relativePath); assertTrue(relativePath, orig.exists()); assertEquals(relativePath, orig.isDirectory(), f.isDirectory()); assertEquals(relativePath, orig.isFile(), f.isFile()); if (Unix.isOperational()) { final Stat fStat = Unix.getLinkInfo(f.getPath(), true); final Stat origStat = Unix.getLinkInfo(orig.getPath(), true); assertEquals(relativePath, origStat.isSymbolicLink(), fStat.isSymbolicLink()); assertEquals(relativePath, origStat.tryGetSymbolicLink(), fStat.tryGetSymbolicLink()); } if (f.isDirectory()) { checkDirectoryEntries(orig, f, entryCount); } } } @Test public void testRoundtripArtificalRootOK() throws IOException { final File dir = createTestDirectory(); final File h5arfile = new File(workingDirectory, "testRoundtripArtificalRootOK.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); HDF5ArchiverFactory.open(h5arfile).archiveFromFilesystem("ttt", dir).close(); final IHDF5ArchiveReader ar = HDF5ArchiverFactory.openForReading(h5arfile); final List list = ar.list("/"); assertEquals(7, list.size()); assertEquals("/ttt", list.get(0).getPath()); assertEquals("/ttt/test", list.get(1).getPath()); assertEquals("/ttt/test/dir_somedir", list.get(2).getPath()); assertEquals("/ttt/test/dir_somedir/file_test2.txt", list.get(3).getPath()); assertEquals("/ttt/test/dir_somedir/link_todir3", list.get(4).getPath()); assertEquals("/ttt/test/dir_someotherdir", list.get(5).getPath()); assertEquals("/ttt/test/file_test1.txt", list.get(6).getPath()); assertEquals("Line 1\nLine 2\nLine 3\n", new String(ar.extractFileAsByteArray("/ttt/test/file_test1.txt"))); assertEquals("A\nB\nC\n", new String(ar.extractFileAsByteArray("/ttt/test/dir_somedir/file_test2.txt"))); assertTrue(ar.test().isEmpty()); List verifyErrors = ar.verifyAgainstFilesystem("/", dir.getParentFile(), "/ttt"); assertTrue(verifyErrors.toString(), verifyErrors.isEmpty()); final List list2 = ar.list("/ttt/test/dir_somedir"); assertEquals(2, list2.size()); assertEquals("file_test2.txt", list2.get(0).getName()); assertEquals("link_todir3", list2.get(1).getName()); final List list3 = ar.list("/ttt/test/dir_somedir", ListParameters.build() .includeTopLevelDirectoryEntry().get()); assertEquals(3, list3.size()); assertEquals("dir_somedir", list3.get(0).getName()); assertEquals("file_test2.txt", list3.get(1).getName()); assertEquals("link_todir3", list3.get(2).getName()); ar.close(); } @Test public void testRoundtripArtificalRootWhichExistsOnFSOK() throws IOException { final long now = System.currentTimeMillis(); final long dirLastChanged = now - 1000L * 3600L * 24 * 3; final File dir = createTestDirectory("ttt", dirLastChanged); // Set some special last modified time and access mode that we can recognize dir.getParentFile().setLastModified(111000L); Unix.setAccessMode(dir.getParent(), (short) 0777); final File h5arfile = new File(workingDirectory, "testRoundtripArtificalRootWhichExistsOnFSOK.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); HDF5ArchiverFactory.open(h5arfile).archiveFromFilesystem("ttt", dir).close(); final IHDF5ArchiveReader ar = HDF5ArchiverFactory.openForReading(h5arfile); final List list = ar.list("/"); assertEquals(7, list.size()); assertEquals("/ttt", list.get(0).getPath()); // Does the archive entry have the last modified time and access mode we have set in the // filesystem? assertEquals(111, list.get(0).getLastModified()); assertEquals((short) 0777, list.get(0).getPermissions()); assertEquals("/ttt/test", list.get(1).getPath()); assertEquals("/ttt/test/dir_somedir", list.get(2).getPath()); assertEquals("/ttt/test/dir_somedir/file_test2.txt", list.get(3).getPath()); assertEquals("/ttt/test/dir_somedir/link_todir3", list.get(4).getPath()); assertEquals("/ttt/test/dir_someotherdir", list.get(5).getPath()); assertEquals("/ttt/test/file_test1.txt", list.get(6).getPath()); assertEquals("Line 1\nLine 2\nLine 3\n", new String(ar.extractFileAsByteArray("/ttt/test/file_test1.txt"))); assertEquals("A\nB\nC\n", new String(ar.extractFileAsByteArray("/ttt/test/dir_somedir/file_test2.txt"))); assertTrue(ar.test().isEmpty()); List verifyErrors = ar.verifyAgainstFilesystem("/", dir.getParentFile(), "/ttt"); assertTrue(verifyErrors.toString(), verifyErrors.isEmpty()); ar.close(); } @Test public void testGetInfo() throws IOException { final File dir = createTestDirectory(); final File h5arfile = new File(workingDirectory, "testGetInfo.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); HDF5ArchiverFactory.open(h5arfile).archiveFromFilesystem(dir).close(); final IHDF5ArchiveReader ar = HDF5ArchiverFactory.openForReading(h5arfile); assertTrue(ar.exists("file_test1.txt")); assertTrue(ar.isRegularFile("file_test1.txt")); assertFalse(ar.isDirectory("file_test1.txt")); assertFalse(ar.isSymLink("file_test1.txt")); assertTrue(ar.exists("dir_somedir")); assertFalse(ar.isRegularFile("dir_somedir")); assertFalse(ar.isSymLink("dir_somedir")); assertTrue(ar.isDirectory("dir_somedir")); assertTrue(ar.exists("dir_somedir/link_todir3")); assertFalse(ar.isRegularFile("dir_somedir/link_todir3")); assertFalse(ar.isDirectory("dir_somedir/link_todir3")); assertTrue(ar.isSymLink("dir_somedir/link_todir3")); assertEquals("../dir_someotherdir", ar.tryGetEntry("dir_somedir/link_todir3", true) .getLinkTarget()); ar.close(); } private static final String ARCHIVE_LISTING = "775\t1001\t1001\t DIR\t2015-02-21 14:01:31\t \t/tmp\n" + "775\t1001\t1001\t DIR\t2015-02-21 14:01:40\t \t/tmp/c\n" + "664\t1001\t1001\t 7\t2015-02-21 14:01:40\t046d0418\t/tmp/c/d\n" + "664\t1001\t1001\t 7\t2015-02-21 14:01:21\t791af05d\t/tmp/a\n" + "664\t1001\t1001\t 7\t2015-02-21 14:01:27\t5237a39e\t/tmp/b"; @Test public void testReadExistingArchive() { final File h5arfile = new File(srcDirectory, "test.h5ar"); System.out.println(h5arfile); assertTrue(h5arfile.exists()); final IHDF5ArchiveReader ar = HDF5ArchiverFactory.openForReading(h5arfile); assertEquals(ARCHIVE_LISTING, toStringNumeric(ar)); final List testResult = ar.test(); assertTrue(ArrayUtils.toString(testResult), testResult.isEmpty()); ar.close(); } @Test public void testReadLegacy_14_12_0_Archive() { final File h5arfile = new File(srcDirectory, "test_14_12_0.h5ar"); System.out.println(h5arfile); assertTrue(h5arfile.exists()); final IHDF5ArchiveReader ar = HDF5ArchiverFactory.openForReading(h5arfile); assertEquals(ARCHIVE_LISTING, toStringNumeric(ar)); final List testResult = ar.test(); assertTrue(ArrayUtils.toString(testResult), testResult.isEmpty()); ar.close(); } private String toStringNumeric(final IHDF5ArchiveReader ar) { final StringBuilder b = new StringBuilder(); for (ArchiveEntry e : ar.list()) { b.append(e.describeLink(true, true)); b.append('\n'); } b.setLength(b.length() - 1); return b.toString(); } private void checkSorted(List entries) { boolean dirs = true; for (int i = 1; i < entries.size(); ++i) { if (dirs && entries.get(i).isDirectory() == false) { dirs = false; } else { assertTrue(entries.get(i - 1).getName().compareTo(entries.get(i).getName()) < 0); } } } @Test public void testManyFiles() { workingDirectory.mkdirs(); final File h5arfile = new File(workingDirectory, "testManyFiles.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); final IHDF5Archiver a = HDF5ArchiverFactory.open(h5arfile); for (int i = 999; i >= 0; --i) { a.archiveFile(Integer.toString(i), new byte[0]); } a.archiveSymlink("symlink", "500"); a.archiveDirectory(NewArchiveEntry.directory("/dir")); a.archiveFile("dir/hello", "hello world".getBytes()); final List entries = a.list("/"); assertEquals(1003, entries.size()); final ArchiveEntry symLinkEntry = a.tryGetEntry("symlink", true); assertNotNull(symLinkEntry); assertTrue(symLinkEntry.isSymLink()); assertTrue(symLinkEntry.hasLinkTarget()); assertEquals("500", symLinkEntry.getLinkTarget()); final ArchiveEntry dirEntry = a.tryGetEntry("dir", true); assertNotNull(dirEntry); assertTrue(dirEntry.isDirectory()); assertFalse(dirEntry.isRegularFile()); assertFalse(dirEntry.isSymLink()); final List entriesDir = a.list("/dir"); assertEquals(1, entriesDir.size()); assertEquals("hello", entriesDir.get(0).getName()); a.close(); final IHDF5ArchiveReader ra = HDF5ArchiverFactory.openForReading(h5arfile); final List entriesRead = ra.list("/", ListParameters.build().nonRecursive().get()); assertEquals(1002, entriesRead.size()); checkSorted(entriesRead); for (int i = 1; i < entriesRead.size() - 1; ++i) { assertTrue(entriesRead.get(i).isRegularFile()); } assertTrue(entriesRead.get(0).isDirectory()); assertTrue(entriesRead.get(entriesRead.size() - 1).isSymLink()); for (int i = 1; i < 1001; ++i) { assertTrue(ra.isRegularFile(Integer.toString(i - 1))); assertFalse(ra.isDirectory(Integer.toString(i - 1))); assertFalse(ra.isSymLink(Integer.toString(i - 1))); } assertTrue(ra.isSymLink("symlink")); assertFalse(ra.isDirectory("symlink")); assertFalse(ra.isRegularFile("symlink")); assertEquals("500", ra.tryGetEntry("symlink", true).getLinkTarget()); assertTrue(ra.isDirectory("dir")); assertFalse(ra.isSymLink("dir")); assertFalse(ra.isRegularFile("dir")); ra.close(); } @Test public void testFollowSymbolicLinks() { workingDirectory.mkdirs(); final File h5arfile = new File(workingDirectory, "testFollowSymbolicLinks.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); final IHDF5Archiver a = HDF5ArchiverFactory.open(h5arfile); a.archiveDirectory(NewArchiveEntry.directory("aDir")); a.archiveFile(NewArchiveEntry.file("aDir/aFile"), "Some file content".getBytes()); a.archiveSymlink(NewArchiveEntry.symlink("aLinkToAFile", "aDir/aFile")); a.archiveSymlink(NewArchiveEntry.symlink("aLinkToADir", "aDir")); a.close(); final IHDF5ArchiveReader ra = HDF5ArchiverFactory.openForReading(h5arfile); final List entries = ra.list("/", ListParameters.build() ./* resolveSymbolicLinks(). */followSymbolicLinks().get()); assertEquals(5, entries.size()); assertEquals("/aDir", entries.get(0).getPath()); assertTrue(entries.get(0).isDirectory()); assertEquals("/aDir/aFile", entries.get(1).getPath()); assertTrue(entries.get(1).isRegularFile()); assertEquals("/aLinkToADir", entries.get(2).getPath()); assertTrue(entries.get(2).isSymLink()); assertEquals("/aLinkToADir/aFile", entries.get(3).getPath()); assertTrue(entries.get(3).isRegularFile()); assertEquals("/aLinkToAFile", entries.get(4).getPath()); assertTrue(entries.get(4).isSymLink()); ra.close(); } @Test public void testFollowAndResolveSymbolicLinks() { workingDirectory.mkdirs(); final File h5arfile = new File(workingDirectory, "testFollowAndResolveSymbolicLinks.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); final IHDF5Archiver a = HDF5ArchiverFactory.open(h5arfile); a.archiveDirectory(NewArchiveEntry.directory("aDir")); a.archiveFile(NewArchiveEntry.file("aDir/aFile"), "Some file content".getBytes()); a.archiveSymlink(NewArchiveEntry.symlink("aLinkToAFile", "aDir/aFile")); a.archiveSymlink(NewArchiveEntry.symlink("aLinkToADir", "aDir")); a.close(); final IHDF5ArchiveReader ra = HDF5ArchiverFactory.openForReading(h5arfile); final List entries = ra.list("/", ListParameters.build().resolveSymbolicLinks().followSymbolicLinks() .get()); assertEquals(5, entries.size()); assertEquals("/aDir", entries.get(0).getPath()); assertTrue(entries.get(0).isDirectory()); assertEquals("/aDir/aFile", entries.get(1).getPath()); assertTrue(entries.get(1).isRegularFile()); assertEquals("/aLinkToADir", entries.get(2).getPath()); assertEquals("/aDir", entries.get(2).getRealPath()); assertTrue(entries.get(2).isDirectory()); assertEquals("/aLinkToADir/aFile", entries.get(3).getPath()); assertTrue(entries.get(3).isRegularFile()); assertEquals("/aLinkToAFile", entries.get(4).getPath()); assertEquals("/aDir/aFile", entries.get(4).getRealPath()); assertTrue(entries.get(4).isRegularFile()); ra.close(); } @Test public void testResolveLinks() { workingDirectory.mkdirs(); final File h5arfile = new File(workingDirectory, "testResolveLinks.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); final IHDF5Archiver a = HDF5ArchiverFactory.open(h5arfile); a.archiveFile(NewArchiveEntry.file("aFile"), "Some file content".getBytes()); a.archiveDirectory(NewArchiveEntry.directory("aDir")); a.archiveSymlink(NewArchiveEntry.symlink("aLinkToAFile", "aFile")); a.archiveSymlink(NewArchiveEntry.symlink("aLinkToADir", "aDir")); a.archiveSymlink(NewArchiveEntry.symlink("aNonsenseLink", "../outOfFS")); a.archiveSymlink(NewArchiveEntry.symlink("aLinkToANonexistingFile", "nonexistingFile")); a.archiveSymlink(NewArchiveEntry.symlink("aDir/aLinkToALinkToAFile", "../aLinkToAFile")); a.archiveSymlink(NewArchiveEntry.symlink("aDir/aLinkToALinkToADir", "/aLinkToADir")); // A loop a.archiveDirectory(NewArchiveEntry.directory("z")); a.archiveSymlink(NewArchiveEntry.symlink("z/y", "..")); a.close(); final IHDF5ArchiveReader ra = HDF5ArchiverFactory.openForReading(h5arfile); // A file is resolved to itself final ArchiveEntry aFileLink = ra.tryGetEntry("aFile", false); assertEquals(aFileLink, ra.tryResolveLink(aFileLink)); // A directory is resolved to itself final ArchiveEntry aDirLink = ra.tryGetEntry("aDir", false); assertEquals(aDirLink, ra.tryResolveLink(aDirLink)); // A symlink to a file is correctly resolved... final ArchiveEntry aSymLinkToAFile = ra.tryGetEntry("aLinkToAFile", true); final ArchiveEntry aResolvedLinkToAFile = ra.tryResolveLink(aSymLinkToAFile); assertNotNull(aResolvedLinkToAFile); assertEquals(aFileLink.getPath(), aResolvedLinkToAFile.getPath()); // .. even when the link target was not read final ArchiveEntry aSymLinkToAFileWithoutTarget = ra.tryGetEntry("aLinkToAFile", false); final ArchiveEntry aResolvedLinkToAFileWithoutTarget = ra.tryResolveLink(aSymLinkToAFileWithoutTarget); assertNotNull(aResolvedLinkToAFileWithoutTarget); assertEquals(aFileLink.getPath(), aResolvedLinkToAFileWithoutTarget.getPath()); // A symlink to a dir is correctly resolved as well final ArchiveEntry aSymLinkToADir = ra.tryGetEntry("aLinkToADir", true); final ArchiveEntry aResolvedLinkToADir = ra.tryResolveLink(aSymLinkToADir); assertNotNull(aResolvedLinkToADir); assertEquals(aDirLink.getPath(), aResolvedLinkToADir.getPath()); // A nonsense link ('/../outOfFS') is resolved to null assertNull(ra.tryResolveLink(ra.tryGetEntry("aNonsenseLink", true))); // A link to a non-existing file is resolved to null assertNull(ra.tryResolveLink(ra.tryGetEntry("aLinkToANonexistingFile", true))); // A link to a link to a file final ArchiveEntry aSymLinkToALinkToAFile = ra.tryGetEntry("/aDir/aLinkToALinkToAFile", false); final ArchiveEntry aResolvedSymLinkToALinkToAFile = ra.tryResolveLink(aSymLinkToALinkToAFile); assertNotNull(aResolvedSymLinkToALinkToAFile); assertEquals(aFileLink.getPath(), aResolvedSymLinkToALinkToAFile.getPath()); final ArchiveEntry aSymLinkToALinkToAFileWithPathInfoKept = ra.tryGetResolvedEntry("/aDir/aLinkToALinkToAFile", true); assertEquals("/aDir", aSymLinkToALinkToAFileWithPathInfoKept.getParentPath()); assertEquals("aLinkToALinkToAFile", aSymLinkToALinkToAFileWithPathInfoKept.getName()); assertTrue(aSymLinkToALinkToAFileWithPathInfoKept.isRegularFile()); assertEquals(ra.tryGetEntry("aFile", false).getSize(), aSymLinkToALinkToAFileWithPathInfoKept.getSize()); // A link to a link to a dir final ArchiveEntry aSymLinkToALinkToADir = ra.tryGetEntry("/aDir/aLinkToALinkToADir", false); final ArchiveEntry aResolvedSymLinkToALinkToADir = ra.tryResolveLink(aSymLinkToALinkToADir); assertNotNull(aResolvedSymLinkToALinkToADir); assertEquals(aDirLink.getPath(), aResolvedSymLinkToALinkToADir.getPath()); final List entries = ra.list("/", ListParameters.build().resolveSymbolicLinks().get()); assertEquals(8, entries.size()); assertEquals("/aDir", entries.get(0).getPath()); assertTrue(entries.get(0).isDirectory()); assertEquals("/aDir/aLinkToALinkToADir", entries.get(1).getPath()); assertEquals("/aDir", entries.get(1).getRealPath()); assertTrue(entries.get(1).isDirectory()); assertEquals("/aDir/aLinkToALinkToAFile", entries.get(2).getPath()); assertEquals("/aFile", entries.get(2).getRealPath()); assertTrue(entries.get(2).isRegularFile()); assertEquals("/z", entries.get(3).getPath()); assertTrue(entries.get(3).isDirectory()); assertEquals("/z/y", entries.get(4).getPath()); assertEquals("/", entries.get(4).getRealPath()); assertTrue(entries.get(4).isDirectory()); assertEquals("/aFile", entries.get(5).getPath()); assertTrue(entries.get(5).isRegularFile()); assertEquals("/aLinkToADir", entries.get(6).getPath()); assertTrue(entries.get(6).isDirectory()); assertEquals("/aLinkToAFile", entries.get(7).getPath()); assertTrue(entries.get(7).isRegularFile()); assertEquals(entries.get(5).getCrc32(), entries.get(7).getCrc32()); assertEquals("/", ra.tryGetResolvedEntry("z/y", false).getPath()); ra.close(); } @Test public void testResolveLinksWithLoops() { workingDirectory.mkdirs(); final File h5arfile = new File(workingDirectory, "testResolveLinksWithLoops.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); final IHDF5Archiver a = HDF5ArchiverFactory.open(h5arfile); a.archiveSymlink(NewArchiveEntry.symlink("a", "b")); a.archiveSymlink(NewArchiveEntry.symlink("b", "a")); a.archiveSymlink(NewArchiveEntry.symlink("c", "d")); a.archiveSymlink(NewArchiveEntry.symlink("d", "e")); a.archiveSymlink(NewArchiveEntry.symlink("e", "c")); a.close(); final IHDF5ArchiveReader ra = HDF5ArchiverFactory.openForReading(h5arfile); assertNull(ra.tryGetResolvedEntry("a", false)); assertNull(ra.tryGetResolvedEntry("d", false)); assertTrue(ra.list("/", ListParameters.build().resolveSymbolicLinks().followSymbolicLinks().get()) .isEmpty()); ra.close(); } @Test(expectedExceptions = ListArchiveTooManySymbolicLinksException.class) public void testResolveLinksWithLoopsInPath() { workingDirectory.mkdirs(); final File h5arfile = new File(workingDirectory, "testResolveLinksWithLoopsInPath.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); final IHDF5Archiver a = HDF5ArchiverFactory.open(h5arfile); // A loop in the paths a.archiveDirectory(NewArchiveEntry.directory("1")); a.archiveDirectory(NewArchiveEntry.directory("2")); a.archiveSymlink(NewArchiveEntry.symlink("1/3", "/2")); a.archiveSymlink(NewArchiveEntry.symlink("2/4", "/1")); a.close(); final IHDF5ArchiveReader ra = HDF5ArchiverFactory.openForReading(h5arfile); try { // Will throw ListArchiveTooManySymbolicLinksException ra.list("/", ListParameters.build().resolveSymbolicLinks().followSymbolicLinks().get()); } finally { ra.close(); } } @Test public void testIntegrityCheck() throws IOException { final File dir = createTestDirectory(); final File h5arfile = new File(workingDirectory, "testIntegrityCheck.h5ar"); h5arfile.delete(); h5arfile.deleteOnExit(); HDF5ArchiverFactory.open(h5arfile).archiveFromFilesystem("ttt", dir).close(); IHDF5Writer w = HDF5Factory.open(h5arfile); w.string().write("/ttt/test/file_test1.txt", "changed behind the back."); w.int8().writeArray("/ttt/test/dir_somedir/file_test2.txt", "A\nB\nD\n".getBytes()); w.close(); final IHDF5ArchiveReader ar = HDF5ArchiverFactory.openForReading(h5arfile); final List failed = ar.test(); ar.close(); assertEquals(2, failed.size()); assertEquals("/ttt/test/dir_somedir/file_test2.txt", failed.get(0).getPath()); assertFalse(failed.get(0).isOK()); assertTrue(failed.get(0).sizeOK()); assertFalse(failed.get(0).checksumOK()); assertEquals("/ttt/test/file_test1.txt", failed.get(1).getPath()); assertFalse(failed.get(1).isOK()); assertFalse(failed.get(1).sizeOK()); assertFalse(failed.get(1).checksumOK()); } }