package net.noderunner.amazon.s3; // This software code is made available "AS IS" without warranties of any // kind. You may copy, display, modify and redistribute the software // code either by itself or as incorporated into your code; provided that // you do not remove any proprietary notices. Your use of this software // code is at your own risk and you waive any claim against Amazon // Digital Services, Inc. or its affiliates with respect to your use of // this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its // affiliates. import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.util.Arrays; import java.util.List; import net.noderunner.amazon.s3.Bucket; import net.noderunner.amazon.s3.CallingFormat; import net.noderunner.amazon.s3.Connection; import net.noderunner.amazon.s3.Entry; import net.noderunner.amazon.s3.GetResponse; import net.noderunner.amazon.s3.GetStreamResponse; import net.noderunner.amazon.s3.Headers; import net.noderunner.amazon.s3.ListAllBucketsResponse; import net.noderunner.amazon.s3.ListResponse; import net.noderunner.amazon.s3.Method; import net.noderunner.amazon.s3.QueryGenerator; import net.noderunner.amazon.s3.Response; import net.noderunner.amazon.s3.S3Object; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.URI; import org.apache.commons.httpclient.methods.EntityEnclosingMethod; import org.apache.commons.httpclient.methods.StringRequestEntity; import org.junit.Before; import org.junit.Test; public class S3Test { String awsAccessKeyId = System.getProperty("accessKey"); String awsSecretAccessKey = System.getProperty("secretKey"); boolean emulated = Boolean.valueOf(System.getProperty("emulated", "true")); Bucket bucket; static final int UnspecifiedMaxKeys = -1; @Before public void setUp() { if (emulated) return; if (awsAccessKeyId == null) throw new IllegalStateException("accessKey system propery null"); if (awsSecretAccessKey == null) throw new IllegalStateException("secretKey system propery null"); // for subdomains (bucket.s3.amazonaws.com), // the bucket name must be lowercase since DNS is case-insensitive bucket = new Bucket(awsAccessKeyId.toLowerCase() + "-test-bucket"); } @Test public void testMe() throws Exception { if (emulated) return; // test all operation for both regular and vanity domains // regular: http://s3.amazonaws.com/key // subdomain: http://bucket.s3.amazonaws.com/key // testing pure vanity domains (http:///key) is not covered here // but is possible with some assitional setup test(CallingFormat.SUBDOMAIN, Connection.LOCATION_DEFAULT, false, Connection.DEFAULT_HOST); test(CallingFormat.PATH, Connection.LOCATION_DEFAULT, true, Connection.DEFAULT_HOST); test(CallingFormat.SUBDOMAIN, Connection.LOCATION_EU, true, Connection.DEFAULT_HOST); } private void test(CallingFormat format, String location, boolean secure, String server) throws Exception { System.out.println((secure ? "http" : "https") + " / " + server + " / " + ((location == null) ? "" : location) + " / " + format.getClass().getName()); Connection conn = new Connection(awsAccessKeyId, awsSecretAccessKey, secure, server, format); QueryGenerator generator = new QueryGenerator(awsAccessKeyId, awsSecretAccessKey, secure, server, format); Response response = conn.create(bucket, location, null); response.assertOk(); ListResponse listBucketResponse = conn.list(bucket); listBucketResponse.assertOk(); for (Entry entry : listBucketResponse.getEntries()) { Response delete = conn.delete(bucket, entry.getKey()); System.out.println("DEL " + delete); delete.assertOk(); } listBucketResponse = conn.list(bucket); assertEquals("list wasn't empty " + listBucketResponse, 0, listBucketResponse.getEntries().size()); System.out.println(listBucketResponse); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", UnspecifiedMaxKeys, null, false, null); // start delimiter tests final String text = "this is a test"; final String key = "example.txt"; final String innerKey = "test/inner.txt"; final String lastKey = "z-last-key.txt"; response = conn.put(bucket, key, new S3Object(text)); response.assertOk(); response = conn.put(bucket, innerKey, new S3Object(text)); response.assertOk(); response = conn.put(bucket, lastKey, new S3Object(text)); response.assertOk(); // plain list listBucketResponse = conn.list(bucket); listBucketResponse.assertOk(); assertEquals("Unexpected list size", 3, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", UnspecifiedMaxKeys, null, false, null); System.out.println("LIST " + listBucketResponse.getEntries()); // root "directory" listBucketResponse = conn.list(bucket, null, null, null, "/", null); listBucketResponse.assertOk(); assertEquals("Unexpected list size " + listBucketResponse, 2, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 1, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", UnspecifiedMaxKeys, "/", false, null); // root "directory" with a max-keys of "1" listBucketResponse = conn.list(bucket, null, null, 1, "/", null); listBucketResponse.assertOk(); assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", 1, "/", true, "example.txt"); // root "directory" with a max-keys of "2" listBucketResponse = conn.list(bucket, null, null, 2, "/", null); listBucketResponse.assertOk(); assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 1, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", 2, "/", true, "test/"); String marker = listBucketResponse.getNextMarker(); listBucketResponse = conn.list(bucket, null, marker, 2, "/", null); listBucketResponse.assertOk(); assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "", marker, 2, "/", false, null); // test "directory" listBucketResponse = conn.list(bucket, "test/", null, null, "/", null); listBucketResponse.assertOk(); assertEquals("Unexpected list size", 1, listBucketResponse.getEntries().size()); assertEquals("Unexpected common prefix size", 0, listBucketResponse.getCommonPrefixEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "test/", "", UnspecifiedMaxKeys, "/", false, null); // remove innerkey response = conn.delete(bucket, innerKey, null); assertEquals( "couldn't delete entry", HttpURLConnection.HTTP_NO_CONTENT, response.getResponseCode()); // remove last key response = conn.delete(bucket, lastKey, null); assertEquals( "couldn't delete entry", HttpURLConnection.HTTP_NO_CONTENT, response.getResponseCode()); // end delimiter tests response = conn.put(bucket, key, new S3Object(text.getBytes(), null), null); response.assertOk(); Headers metadata = new Headers(); metadata.put("title", "title"); response = conn.put(bucket, key, new S3Object(text.getBytes(), metadata), null); response.assertOk(); GetResponse getResponse = conn.get(bucket, key, null); getResponse.assertOk(); assertEquals("didn't get the right data back", text.getBytes(), getResponse.getObject().getData()); assertEquals("didn't get the right metadata back", 1, getResponse.getObject().getMetadata().size()); assertEquals( "didn't get the right metadata back", "title", getResponse.getObject().getMetadata().getValue("title")); assertEquals( "didn't get the right content-length", ""+text.length(), getResponse.getHeaderField("Content-Length")); GetStreamResponse streamResponse = conn.getStream(bucket, key); InputStream is = streamResponse.getInputStream(); byte b[] = new byte[text.length()]; int len = is.read(b); assertEquals("didn't get the right data back " + len, text.getBytes(), b); streamResponse.release(); String titleWithSpaces = " \t title with leading and trailing spaces "; Headers h = new Headers(); h.put("title", titleWithSpaces); response = conn.put(bucket, key, new S3Object(text.getBytes(), h), null); assertEquals( "couldn't put metadata with leading and trailing spaces", HttpURLConnection.HTTP_OK, response.getResponseCode()); getResponse = conn.get(bucket, key, null); assertEquals( "couldn't get object", HttpURLConnection.HTTP_OK, getResponse.getResponseCode()); assertEquals("didn't get the right metadata back", getResponse.getObject().getMetadata().size(), 1); assertEquals( "didn't get the right metadata back", titleWithSpaces.trim(), getResponse.getObject().getMetadata().getValue("title")); String weirdKey = "&=//%# ++++"; response = conn.put(bucket, weirdKey, new S3Object(text.getBytes())); assertEquals( "couldn't put weird key", HttpURLConnection.HTTP_OK, response.getResponseCode()); getResponse = conn.get(bucket, weirdKey, null); assertEquals( "couldn't get weird key", HttpURLConnection.HTTP_OK, getResponse.getResponseCode()); // start acl test getResponse = conn.getACL(bucket, key, null); assertEquals( "couldn't get acl", HttpURLConnection.HTTP_OK, getResponse.getResponseCode()); byte[] acl = getResponse.getObject().getData(); response = conn.putACL(bucket, key, new String(acl), null); assertEquals( "couldn't put acl", HttpURLConnection.HTTP_OK, response.getResponseCode()); getResponse = conn.getACL(bucket, null); assertEquals( "couldn't get bucket acl", HttpURLConnection.HTTP_OK, getResponse.getResponseCode()); byte[] bucketACL = getResponse.getObject().getData(); response = conn.putACL(bucket, new String(bucketACL), null); assertEquals( "couldn't put bucket acl", HttpURLConnection.HTTP_OK, response.getResponseCode()); // end acl test // bucket logging tests getResponse = conn.getBucketLogging(bucket, null); assertEquals( "couldn't get bucket logging config", HttpURLConnection.HTTP_OK, getResponse.getResponseCode()); byte[] bucketLogging = getResponse.getObject().getData(); response = conn.putBucketLogging(bucket, new String(bucketLogging), null); assertEquals( "couldn't put bucket logging config", HttpURLConnection.HTTP_OK, response.getResponseCode()); // end bucket logging tests listBucketResponse = conn.list(bucket, null, null, null, null); assertEquals( "couldn't list bucket", HttpURLConnection.HTTP_OK, listBucketResponse.getResponseCode()); List entries = listBucketResponse.getEntries(); assertEquals("didn't get back the right number of entries", 2, entries.size()); // depends on weirdKey < $key assertEquals("first key isn't right", weirdKey, ((Entry)entries.get(0)).getKey()); assertEquals("second key isn't right", key, ((Entry)entries.get(1)).getKey()); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", UnspecifiedMaxKeys, null, false, null); listBucketResponse = conn.list(bucket, null, null, 1, null); assertEquals( "couldn't list bucket", HttpURLConnection.HTTP_OK, listBucketResponse.getResponseCode()); assertEquals( "didn't get back the right number of entries", 1, listBucketResponse.getEntries().size()); verifyBucketResponseParameters(listBucketResponse, bucket, "", "", 1, null, true, null); for (Entry entry : entries) { response = conn.delete(bucket, entry.getKey(), null); assertEquals( "couldn't delete entry", HttpURLConnection.HTTP_NO_CONTENT, response.getResponseCode()); } ListAllBucketsResponse listAllMyBucketsResponse = conn.listAllBuckets(); assertEquals( "couldn't list all my buckets", HttpURLConnection.HTTP_OK, listAllMyBucketsResponse.getResponseCode()); List buckets = listAllMyBucketsResponse.getEntries(); response = conn.delete(bucket); assertEquals( "couldn't delete bucket", HttpURLConnection.HTTP_NO_CONTENT, response.getResponseCode()); listAllMyBucketsResponse = conn.listAllBuckets(); assertEquals( "couldn't list all my buckets", HttpURLConnection.HTTP_OK, listAllMyBucketsResponse.getResponseCode()); assertEquals( "bucket count is incorrect", buckets.size() - 1, listAllMyBucketsResponse.getEntries().size()); checkURI( generator.create(bucket, null), Method.PUT, HttpURLConnection.HTTP_OK, "couldn't create bucket"); checkURI( generator.put(bucket, key, new S3Object("test data".getBytes(), null), null), Method.PUT, HttpURLConnection.HTTP_OK, "put object", "test data"); checkURI( generator.get(bucket, key, null), Method.GET, HttpURLConnection.HTTP_OK, "get object"); checkURI( generator.list(bucket, null, null, null, null), Method.GET, HttpURLConnection.HTTP_OK, "list bucket"); checkURI( generator.listAllBuckets(), Method.GET, HttpURLConnection.HTTP_OK, "list all my buckets"); checkURI( generator.getACL(bucket, key, null), Method.GET, HttpURLConnection.HTTP_OK, "get acl"); checkURI( generator.putACL(bucket, key, null), Method.PUT, HttpURLConnection.HTTP_OK, "put acl", new String(acl)); checkURI( generator.getACL(bucket, null), Method.GET, HttpURLConnection.HTTP_OK, "get bucket acl"); checkURI( generator.putACL(bucket, null), Method.PUT, HttpURLConnection.HTTP_OK, "put bucket acl", new String(bucketACL)); checkURI( generator.getBucketLogging(bucket, null), Method.GET, HttpURLConnection.HTTP_OK, "get bucket logging"); checkURI( generator.putBucketLogging(bucket, null), Method.PUT, HttpURLConnection.HTTP_OK, "put bucket logging", new String(bucketLogging)); checkURI( generator.delete(bucket, key, null), Method.DELETE, HttpURLConnection.HTTP_NO_CONTENT, "delete object"); checkURI( generator.delete(bucket, null), Method.DELETE, HttpURLConnection.HTTP_NO_CONTENT, "delete bucket"); } private static void verifyBucketResponseParameters( ListResponse listBucketResponse, Bucket bucket, String prefix, String marker, int maxKeys, String delimiter, boolean isTruncated, String nextMarker ) { assertEquals("Bucket name should match.", bucket.getName(), listBucketResponse.getName()); assertEquals("Bucket prefix should match.", prefix, listBucketResponse.getPrefix()); assertEquals("Bucket marker should match.", marker, listBucketResponse.getMarker()); assertEquals("Bucket delimiter should match.", delimiter, listBucketResponse.getDelimiter()); if ( UnspecifiedMaxKeys != maxKeys ) { assertEquals("Bucket max-keys should match.", maxKeys, listBucketResponse.getMaxKeys()); } assertEquals("Bucket should not be truncated.", isTruncated, listBucketResponse.isTruncated()); assertEquals("Bucket nextMarker should match.", nextMarker, listBucketResponse.getNextMarker()); } private static void assertEquals(String message, int expected, int actual) { if (expected != actual) { throw new RuntimeException(message + ": expected " + expected + " but got " + actual); } } private static void assertEquals(String message, byte[] expected, byte[] actual) { if (! Arrays.equals(expected, actual)) { throw new RuntimeException( message + ": expected " + new String(expected) + " but got " + new String(actual)); } } private static void assertEquals(String message, Object expected, Object actual) { if (expected != actual && (actual == null || ! actual.equals(expected))) { throw new RuntimeException(message + ": expected " + expected + " but got " + actual); } } private static void assertEquals(String message, boolean expected, boolean actual) { if (expected != actual) { throw new RuntimeException(message + ": expected " + expected + " but got " + actual); } } private static void checkURI(URI uri, Method method, int code, String message) throws MalformedURLException, IOException { checkURI(uri, method, code, message, null); } private static void checkURI(URI uri, Method method, int code, String message, String data) throws MalformedURLException, IOException { if (data == null) data = ""; HttpClient client = new HttpClient(); HttpMethod httpMethod = method.createHttpMethod(); if (method == Method.PUT) { ((EntityEnclosingMethod) httpMethod).setRequestEntity(new StringRequestEntity(data)); } httpMethod.setURI(uri); int response = client.executeMethod(httpMethod); httpMethod.releaseConnection(); assertEquals(message, code, response); } private void readline() throws IOException { // TODO // System.in.read(); } @Test public void testDriver() throws Exception { if (emulated) return; Bucket bucket = new Bucket(awsAccessKeyId.toLowerCase() + "-test-bucket"); String keyName = "KEY"; Connection conn = new Connection(awsAccessKeyId, awsSecretAccessKey); QueryGenerator generator = new QueryGenerator(awsAccessKeyId, awsSecretAccessKey); // Check if the bucket exists. The high availability engineering of // Amazon S3 is focused on get, put, list, and delete operations. // Because bucket operations work against a centralized, global // resource space, it is not appropriate to make bucket create or // delete calls on the high availability code path of your application. // It is better to create or delete buckets in a separate initialization // or setup routine that you run less often. if (!conn.exists(bucket)) { System.out.println("----- creating bucket -----"); System.out.println(conn.create(bucket, Connection.LOCATION_DEFAULT, null).getResponseMessage()); // sample creating an EU located bucket. // (note path-style urls will not work with location-constrained buckets) //System.out.println(conn.createBucket(bucketName, AWSAuthConnection.LOCATION_EU, null).connection.getResponseMessage()); } System.out.println("----- listing bucket -----"); System.out.println(conn.list(bucket).getEntries()); System.out.println("----- bucket location -----"); System.out.println(conn.getLocation(bucket).getLocation()); System.out.println("----- putting object -----"); S3Object object = new S3Object("this is a test".getBytes(), null); Headers headers = new Headers(); headers.put("Content-Type", "text/plain"); System.out.println( conn.put(bucket, keyName, object, headers).getResponseMessage() ); System.out.println("----- listing bucket -----"); System.out.println(conn.list(bucket, null, null, null, null).getEntries()); System.out.println("----- getting object -----"); System.out.println( new String(conn.get(bucket, keyName, null).getObject().getData()) ); System.out.println("----- query string auth example -----"); generator.setExpiresIn(60 * 1000); System.out.println("Try this url in your web browser (it will only work for 60 seconds)\n"); System.out.println(generator.get(bucket, keyName, null)); System.out.print("\npress enter> "); readline(); System.out.println("\nNow try just the url without the query string arguments. It should fail.\n"); System.out.println(generator.makeBareURI(bucket, keyName)); System.out.print("\npress enter> "); readline(); System.out.println("----- putting object with metadata and public read acl -----"); Headers metadata = new Headers(); metadata.put("blah", "foo"); object = new S3Object("this is a publicly readable test".getBytes(), new Headers(metadata)); headers = new Headers(); headers.put("x-amz-acl", "public-read"); headers.put("Content-Type", "text/plain"); System.out.println( conn.put(bucket, keyName + "-public", object, headers).getResponseMessage() ); System.out.println("----- anonymous read test -----"); System.out.println("\nYou should be able to try this in your browser\n"); System.out.println(generator.makeBareURI(bucket, keyName + "-public")); System.out.print("\npress enter> "); readline(); System.out.println("----- path style url example -----"); System.out.println("\nNon-location-constrained buckets can also be specified as part of the url path. (This was the original url style supported by S3.)"); System.out.println("\nTry this url out in your browser (it will only be valid for 60 seconds)\n"); generator.setCallingFormat(CallingFormat.PATH); // could also have been done like this: // generator = new QueryStringAuthGenerator(awsAccessKeyId, awsSecretAccessKey, true, Utils.DEFAULT_HOST, CallingFormat.getPathCallingFormat()); generator.setExpiresIn(60 * 1000); System.out.println(generator.get(bucket, keyName, null)); System.out.print("\npress enter> "); readline(); System.out.println("----- getting object's acl -----"); System.out.println(new String(conn.getACL(bucket, keyName, null).getObject().getData())); System.out.println("----- deleting objects -----"); System.out.println( conn.delete(bucket, keyName, null).getResponseMessage() ); System.out.println( conn.delete(bucket, keyName + "-public", null).getResponseMessage() ); System.out.println("----- listing bucket -----"); System.out.println(conn.list(bucket, null, null, null, null).getEntries()); System.out.println("----- listing all my buckets -----"); System.out.println(conn.listAllBuckets().getEntries()); System.out.println("----- deleting bucket -----"); System.out.println( conn.delete(bucket).getResponseMessage() ); } }