#!/usr/bin/python -u
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib
import hmac
import json
import time
import six

from copy import deepcopy
from hashlib import md5
from six.moves.urllib.parse import quote, unquote

import test.functional as tf

from swift.common.swob import normalize_etag
from swift.common.utils import MD5_OF_EMPTY_STRING, config_true_value
from swift.common.middleware.versioned_writes.object_versioning import \
    DELETE_MARKER_CONTENT_TYPE

from test.functional.tests import Base, Base2, BaseEnv, Utils
from test.functional import cluster_info, SkipTest
from test.functional.swift_test_client import Connection, \
    ResponseError
from test.functional.test_tempurl import TestContainerTempurlEnv, \
    TestTempurlEnv


def setUpModule():
    tf.setup_package()


def tearDownModule():
    tf.teardown_package()


class TestObjectVersioningEnv(BaseEnv):
    account2 = None
    versions_header_key = 'X-Versions-Enabled'

    @classmethod
    def setUp(cls):
        super(TestObjectVersioningEnv, cls).setUp()

        if not tf.skip2:
            # Second connection for ACL tests
            config2 = deepcopy(tf.config)
            config2['account'] = tf.config['account2']
            config2['username'] = tf.config['username2']
            config2['password'] = tf.config['password2']
            cls.conn2 = Connection(config2)
            cls.conn2.authenticate()

        if six.PY2:
            # avoid getting a prefix that stops halfway through an encoded
            # character
            prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
        else:
            prefix = Utils.create_name()[:10]

        cls.container = cls.account.container(prefix + "-objs")
        container_headers = {cls.versions_header_key: 'True'}
        if not cls.container.create(hdrs=container_headers):
            raise ResponseError(cls.conn.response)

        cls.unversioned_container = cls.account.container(
            prefix + "-unversioned")
        if not cls.unversioned_container.create():
            raise ResponseError(cls.conn.response)

        if not tf.skip2:
            # setup another account to test ACLs
            config2 = deepcopy(tf.config)
            config2['account'] = tf.config['account2']
            config2['username'] = tf.config['username2']
            config2['password'] = tf.config['password2']
            cls.conn2 = Connection(config2)
            cls.storage_url2, cls.storage_token2 = cls.conn2.authenticate()
            cls.account2 = cls.conn2.get_account()
            cls.account2.delete_containers()

        if not tf.skip3:
            # setup another account with no access to anything to test ACLs
            config3 = deepcopy(tf.config)
            config3['account'] = tf.config['account']
            config3['username'] = tf.config['username3']
            config3['password'] = tf.config['password3']
            cls.conn3 = Connection(config3)
            cls.storage_url3, cls.storage_token3 = cls.conn3.authenticate()
            cls.account3 = cls.conn3.get_account()

        # the allowed headers are configurable in object server, so we cannot
        # assert that content-encoding or content-disposition get *copied* to
        # the object version unless they were set on the original PUT, so
        # populate expected_headers by making a HEAD on the original object
        precheck_container = cls.account.container('header-precheck-cont')
        if not precheck_container.create():
            raise ResponseError(cls.conn.response)
        test_obj = precheck_container.file('test_allowed_headers')
        put_headers = {'Content-Type': 'text/jibberish01',
                       'Content-Encoding': 'gzip',
                       'Content-Disposition': 'attachment; filename=myfile'}
        test_obj.write(b"aaaaa", hdrs=put_headers)
        test_obj.initialize()
        resp_headers = {
            h.lower(): v for h, v in test_obj.conn.response.getheaders()}
        cls.expected_headers = {}
        for k, v in put_headers.items():
            if k.lower() in resp_headers:
                cls.expected_headers[k] = v
        precheck_container.delete_recursive()

    @classmethod
    def tearDown(cls):
        if cls.account:
            cls.account.delete_containers()
        if cls.account2:
            cls.account2.delete_containers()


class TestObjectVersioningBase(Base):
    env = TestObjectVersioningEnv

    def setUp(self):
        super(TestObjectVersioningBase, self).setUp()
        if 'object_versioning' not in tf.cluster_info:
            raise SkipTest("Object Versioning not enabled")

        self._account_name = None

        # make sure versioning is enabled,
        # since it gets disabled in tearDown
        self.env.container.update_metadata(
            hdrs={self.env.versions_header_key: 'True'})

    def _tear_down_files(self, container):
        try:
            # only delete files and not containers
            # as they were configured in self.env
            # get rid of any versions so they aren't restored

            container.update_metadata(
                hdrs={self.env.versions_header_key: 'False'})

            # get rid of originals
            container.delete_files()

            # delete older versions
            listing_parms = {'versions': None, 'format': 'json'}
            for obj_info in container.files(parms=listing_parms):
                prev_version = container.file(obj_info['name'])
                prev_version.delete(
                    parms={'version-id': obj_info['version_id']})

        except ResponseError:
            pass

    def tearDown(self):
        super(TestObjectVersioningBase, self).tearDown()
        self._tear_down_files(self.env.container)

    def assertTotalVersions(self, container, count):
        listing_parms = {'versions': None}
        self.assertEqual(count, len(container.files(parms=listing_parms)))

    def assertContentTypes(self, container, expected_content_types):
        listing_parms = {'versions': None,
                         'format': 'json',
                         'reverse': 'true'}
        self.assertEqual(expected_content_types, [
            o['content_type']
            for o in container.files(parms=listing_parms)])


class TestObjectVersioning(TestObjectVersioningBase):

    @property
    def account_name(self):
        if not self._account_name:
            self._account_name = self.env.conn.storage_path.rsplit('/', 1)[-1]
        return self._account_name

    def test_disable_version(self):
        # sanity
        self.assertTrue(
            config_true_value(self.env.container.info()['versions_enabled']))

        # disable it
        self.env.container.update_metadata(
            hdrs={self.env.versions_header_key: 'False'})
        self.assertFalse(
            config_true_value(self.env.container.info()['versions_enabled']))

        # enabled it back
        self.env.container.update_metadata(
            hdrs={self.env.versions_header_key: 'True'})
        self.assertTrue(
            config_true_value(self.env.container.info()['versions_enabled']))

    def test_account_list_containers(self):
        cont_listing = self.env.account.containers()
        self.assertEqual(cont_listing, [self.env.container.name,
                                        self.env.unversioned_container.name])
        self.env.account.delete_containers()
        prefix = Utils.create_name()

        def get_name(i):
            return prefix + '-%02d' % i

        num_container = [15, 20]
        for i in range(num_container[1]):
            name = get_name(i)
            container = self.env.account.container(name)
            container.create()

        limit = 5
        cont_listing = self.env.account.containers(parms={'limit': limit})
        self.assertEqual(cont_listing, [get_name(i) for i in range(limit)])

        for i in range(num_container[0], num_container[1]):
            name = get_name(i)
            container = self.env.account.container(name)
            container.update_metadata(
                hdrs={self.env.versions_header_key: 'True'})

        cont_listing = self.env.account.containers(parms={'limit': limit})
        self.assertEqual(cont_listing, [get_name(i) for i in range(limit)])

        # we're in charge of getting everything back to normal
        self.env.account.delete_containers()
        self.env.container.create()
        self.env.unversioned_container.create()

    def assert_previous_version(self, object_name, version_id, content,
                                content_type, expected_headers={},
                                not_expected_header_keys=[],
                                check_env_expected_headers=False):
        '''
        Find previous version of an object using the ?versions API
        then, assert object data and metadata using ?version-id API
        '''
        prev_version = self.env.container.file(object_name)
        prev_version.initialize(parms={'version-id': version_id})
        self.assertEqual(content, prev_version.read(
            parms={'version-id': version_id}))
        self.assertEqual(content_type, prev_version.content_type)
        # make sure the new obj metadata did not leak to the prev. version
        resp_headers = {
            h.lower(): v for h, v in prev_version.conn.response.getheaders()}

        for k in not_expected_header_keys:
            self.assertNotIn(k, resp_headers)

        for k, v in expected_headers.items():
            self.assertIn(k.lower(), resp_headers)
            self.assertEqual(v, resp_headers[k.lower()])

        # also check env expected_headers
        if check_env_expected_headers:
            for k, v in self.env.expected_headers.items():
                self.assertIn(k.lower(), resp_headers)
                self.assertEqual(v, resp_headers[k.lower()])

    def test_expiry(self):
        # sanity
        container = self.env.container
        self.assertTrue(
            config_true_value(self.env.container.info()['versions_enabled']))

        versioned_obj1 = container.file(Utils.create_name())
        put_headers = {'Content-Type': 'text/blah-blah-blah',
                       'X-Delete-After': '1',
                       'X-Object-Meta-Color': 'blue'}
        resp = versioned_obj1.write(b"aaaaa", hdrs=put_headers,
                                    return_resp=True)
        version_id1 = resp.getheader('x-object-version-id')

        versioned_obj2 = container.file(Utils.create_name())
        resp = versioned_obj2.write(b"aaaaa", hdrs={}, return_resp=True)
        version_id2 = resp.getheader('x-object-version-id')

        # swift_test_client's File API doesn't really allow for POSTing
        # arbitrary headers, so...
        def post(url, token, parsed, conn):
            conn.request('POST', '%s/%s/%s' % (parsed.path, container,
                                               versioned_obj2.name),
                         '', {'X-Auth-Token': token,
                              'Content-Length': '0',
                              'X-Object-Meta-Color': 'red',
                              'X-Delete-After': '1'})
            return tf.check_response(conn)
        resp = tf.retry(post)
        resp.read()
        self.assertEqual(resp.status, 202)

        time.sleep(1)

        # Links have expired
        with self.assertRaises(ResponseError) as cm:
            versioned_obj1.info()
        self.assertEqual(404, cm.exception.status)

        with self.assertRaises(ResponseError) as cm:
            versioned_obj2.info()
        self.assertEqual(404, cm.exception.status)

        # But data are still there
        versioned_obj1.initialize(parms={'version-id': version_id1})
        self.assertEqual('text/blah-blah-blah', versioned_obj1.content_type)
        self.assertEqual('blue', versioned_obj1.metadata['color'])

        versioned_obj2.initialize(parms={'version-id': version_id2})
        self.assertEqual('application/octet-stream',
                         versioned_obj2.content_type)
        self.assertEqual('red', versioned_obj2.metadata['color'])

        # Note that links may still show up in listings, depending on how
        # aggressive the object-expirer is. When doing a version-aware
        # listing, though, we'll only ever have the two entries.
        self.assertTotalVersions(container, 2)

    def test_get_if_match(self):
        body = b'data'
        oname = Utils.create_name()
        obj = self.env.unversioned_container.file(oname)
        resp = obj.write(body, return_resp=True)
        etag = resp.getheader('etag')
        self.assertEqual(md5(body).hexdigest(), normalize_etag(etag))

        # un-versioned object is cool with with if-match
        self.assertEqual(body, obj.read(hdrs={'if-match': etag}))
        with self.assertRaises(ResponseError) as cm:
            obj.read(hdrs={'if-match': 'not-the-etag'})
        self.assertEqual(412, cm.exception.status)

        v_obj = self.env.container.file(oname)
        resp = v_obj.write(body, return_resp=True)
        self.assertEqual(resp.getheader('etag'), etag)

        # versioned object is too with with if-match
        self.assertEqual(body, v_obj.read(hdrs={
            'if-match': normalize_etag(etag)}))
        # works quoted, too
        self.assertEqual(body, v_obj.read(hdrs={
            'if-match': '"%s"' % normalize_etag(etag)}))
        with self.assertRaises(ResponseError) as cm:
            v_obj.read(hdrs={'if-match': 'not-the-etag'})
        self.assertEqual(412, cm.exception.status)

    def test_container_acls(self):
        if tf.skip3:
            raise SkipTest('Username3 not set')

        obj = self.env.container.file(Utils.create_name())
        resp = obj.write(b"data", return_resp=True)
        version_id = resp.getheader('x-object-version-id')
        self.assertIsNotNone(version_id)

        with self.assertRaises(ResponseError) as cm:
            obj.read(hdrs={'X-Auth-Token': self.env.conn3.storage_token})
        self.assertEqual(403, cm.exception.status)

        # Container ACLs work more or less like they always have
        self.env.container.update_metadata(
            hdrs={'X-Container-Read': self.env.conn3.user_acl})
        self.assertEqual(b"data", obj.read(hdrs={
            'X-Auth-Token': self.env.conn3.storage_token}))

        # But the version-specifc GET still requires a swift owner
        with self.assertRaises(ResponseError) as cm:
            obj.read(hdrs={'X-Auth-Token': self.env.conn3.storage_token},
                     parms={'version-id': version_id})
        self.assertEqual(403, cm.exception.status)

        # If it's pointing to a symlink that points elsewhere, that still needs
        # to be authed
        tgt_name = Utils.create_name()
        self.env.unversioned_container.file(tgt_name).write(b'link')
        sym_tgt_header = quote(unquote('%s/%s' % (
            self.env.unversioned_container.name, tgt_name)))
        obj.write(hdrs={'X-Symlink-Target': sym_tgt_header})

        # So, user1's good...
        self.assertEqual(b'link', obj.read())
        # ...but user3 can't
        with self.assertRaises(ResponseError) as cm:
            obj.read(hdrs={'X-Auth-Token': self.env.conn3.storage_token})
        self.assertEqual(403, cm.exception.status)

        # unless we add the acl to the unversioned_container
        self.env.unversioned_container.update_metadata(
            hdrs={'X-Container-Read': self.env.conn3.user_acl})
        self.assertEqual(b'link', obj.read(
            hdrs={'X-Auth-Token': self.env.conn3.storage_token}))

    def _test_overwriting_setup(self, obj_name=None):
        # sanity
        container = self.env.container
        self.assertTrue(
            config_true_value(self.env.container.info()['versions_enabled']))

        expected_content_types = []
        self.assertTotalVersions(container, 0)
        obj_name = obj_name or Utils.create_name()

        versioned_obj = container.file(obj_name)
        put_headers = {'Content-Type': 'text/jibberish01',
                       'Content-Encoding': 'gzip',
                       'Content-Disposition': 'attachment; filename=myfile'}
        resp = versioned_obj.write(b"aaaaa", hdrs=put_headers,
                                   return_resp=True)
        v1_version_id = resp.getheader('x-object-version-id')
        expected_content_types.append('text/jibberish01')
        self.assertContentTypes(container, expected_content_types)
        obj_info = versioned_obj.info()
        self.assertEqual('text/jibberish01', obj_info['content_type'])

        self.assertTotalVersions(container, 1)
        resp = versioned_obj.write(
            b"bbbbb",
            hdrs={'Content-Type': 'text/jibberish02',
                  'X-Object-Meta-Foo': 'Bar'},
            return_resp=True)
        v2_version_id = resp.getheader('x-object-version-id')
        versioned_obj.initialize()
        self.assertEqual(versioned_obj.content_type, 'text/jibberish02')
        self.assertEqual(versioned_obj.metadata['foo'], 'Bar')
        resp_headers = {
            h.lower(): v for h, v in versioned_obj.conn.response.getheaders()}
        content_location = quote('/v1/%s/%s/%s' % (
            self.account_name, container.name, obj_name
        )) + '?version-id=%s' % (v2_version_id,)
        self.assertEqual(content_location, resp_headers['content-location'])
        expected_content_types.append('text/jibberish02')
        self.assertContentTypes(container, expected_content_types)

        # the old version got saved off
        self.assertTotalVersions(container, 2)

        self.assert_previous_version(
            obj_name, v1_version_id, b'aaaaa', 'text/jibberish01',
            not_expected_header_keys=['X-Object-Meta-Foo'],
            check_env_expected_headers=True)

        # check that POST does not create a new version
        versioned_obj.sync_metadata(metadata={'fu': 'baz'})
        self.assertTotalVersions(container, 2)

        self.assert_previous_version(
            obj_name, v2_version_id, b'bbbbb', 'text/jibberish02',
            expected_headers={'X-Object-Meta-Fu': 'baz'})

        # if we overwrite it again, there are three versions
        resp = versioned_obj.write(b"ccccc", return_resp=True)
        v3_version_id = resp.getheader('x-object-version-id')
        expected_content_types.append('text/jibberish02')
        self.assertContentTypes(container, expected_content_types)
        self.assertTotalVersions(self.env.container, 3)

        # versioned_obj keeps the newest content
        self.assertEqual(b"ccccc", versioned_obj.read())

        # test copy from a different container
        src_container = self.env.account.container(Utils.create_name())
        self.assertTrue(src_container.create())
        src_name = Utils.create_name()
        src_obj = src_container.file(src_name)
        src_obj.write(b"ddddd", hdrs={'Content-Type': 'text/jibberish04'})
        src_obj.copy(container.name, obj_name)
        expected_content_types.append('text/jibberish04')
        self.assertContentTypes(container, expected_content_types)

        self.assertEqual(b"ddddd", versioned_obj.read())
        versioned_obj.initialize()
        self.assertEqual(versioned_obj.content_type, 'text/jibberish04')

        # make sure versions container has the previous version
        self.assertTotalVersions(self.env.container, 4)
        self.assert_previous_version(
            obj_name, v3_version_id, b'ccccc', 'text/jibberish02')

        # test delete
        # at first, delete will succeed with 204
        versioned_obj.delete()
        expected_content_types.append(
            'application/x-deleted;swift_versions_deleted=1')

        # after that, any time the delete doesn't restore the old version
        # and we will get 404 NotFound
        for x in range(3):
            with self.assertRaises(ResponseError) as cm:
                versioned_obj.delete()
            self.assertEqual(404, cm.exception.status)
            expected_content_types.append(
                'application/x-deleted;swift_versions_deleted=1')

        # finally, we have 4 versioned items and 4 delete markers total in
        # the versions container
        self.assertTotalVersions(self.env.container, 8)
        self.assertContentTypes(self.env.container, expected_content_types)

        # update versioned_obj
        versioned_obj.write(b"eeee", hdrs={'Content-Type': 'text/thanksgiving',
                            'X-Object-Meta-Bar': 'foo'})

        # verify the PUT object is kept successfully
        obj_info = versioned_obj.info()
        self.assertEqual('text/thanksgiving', obj_info['content_type'])

        # 8 plus one more write
        self.assertTotalVersions(self.env.container, 9)

        # update versioned_obj
        versioned_obj.write(b"ffff", hdrs={'Content-Type': 'text/teriyaki',
                            'X-Object-Meta-Food': 'chickin'})

        # verify the PUT object is kept successfully
        obj_info = versioned_obj.info()
        self.assertEqual('text/teriyaki', obj_info['content_type'])

        # 9 plus one more write
        self.assertTotalVersions(self.env.container, 10)

        versioned_obj.delete()
        with self.assertRaises(ResponseError) as cm:
            versioned_obj.read()
        self.assertEqual(404, cm.exception.status)

        # 10 plus delete marker
        self.assertTotalVersions(self.env.container, 11)

        return (versioned_obj, expected_content_types)

    def test_overwriting(self):
        versioned_obj, expected_content_types = \
            self._test_overwriting_setup()

    def test_make_old_version_latest(self):
        obj_name = Utils.create_name()
        versioned_obj = self.env.container.file(obj_name)
        versions = [{
            'content_type': 'text/jibberish01',
            'body': b'aaaaa',
        }, {
            'content_type': 'text/jibberish02',
            'body': b'bbbbbb',
        }, {
            'content_type': 'text/jibberish03',
            'body': b'ccccccc',
        }]
        for version in versions:
            resp = versioned_obj.write(version['body'], hdrs={
                'Content-Type': version['content_type']}, return_resp=True)
            version['version_id'] = resp.getheader('x-object-version-id')
        expected = [{
            'name': obj_name,
            'content_type': version['content_type'],
            'version_id': version['version_id'],
            'hash': md5(version['body']).hexdigest(),
            'bytes': len(version['body'],)
        } for version in reversed(versions)]
        for item, is_latest in zip(expected, (True, False, False)):
            item['is_latest'] = is_latest
        versions_listing = self.env.container.files(parms={
            'versions': 'true', 'format': 'json'})
        for item in versions_listing:
            item.pop('last_modified')
        self.assertEqual(expected, versions_listing)

        versioned_obj.write(b'', parms={
            'version-id': versions[1]['version_id']})
        self.assertEqual(b'bbbbbb', versioned_obj.read())
        for item, is_latest in zip(expected, (False, True, False)):
            item['is_latest'] = is_latest
        versions_listing = self.env.container.files(parms={
            'versions': 'true', 'format': 'json'})
        for item in versions_listing:
            item.pop('last_modified')
        self.assertEqual(expected, versions_listing)

    def test_overwriting_with_url_encoded_object_name(self):
        obj_name = Utils.create_name() + '%25ff'
        versioned_obj, expected_content_types = \
            self._test_overwriting_setup(obj_name)

    def _test_versioning_dlo_setup(self):
        if tf.in_process:
            tf.skip_if_no_xattrs()

        container = self.env.container
        obj_name = Utils.create_name()

        for i in ('1', '2', '3'):
            time.sleep(.01)  # guarantee that the timestamp changes
            obj_name_seg = 'segs_' + obj_name + '/' + i
            versioned_obj = container.file(obj_name_seg)
            versioned_obj.write(i.encode('ascii'))
            # immediately overwrite
            versioned_obj.write((i + i).encode('ascii'))

        # three objects 2 versions each
        self.assertTotalVersions(self.env.container, 6)

        man_file = container.file(obj_name)

        # write a normal file first
        resp = man_file.write(
            b'old content', hdrs={'Content-Type': 'text/jibberish01'},
            return_resp=True)
        v1_version_id = resp.getheader('x-object-version-id')
        self.assertEqual(b'old content', man_file.read())

        # guarantee that the timestamp changes
        time.sleep(.01)

        # overwrite with a dlo manifest
        dlo_prefix = quote(unquote('%s/segs_%s/' % (
            self.env.container.name, obj_name)))
        resp = man_file.write(
            b'', hdrs={'Content-Type': 'text/jibberish02',
                       'X-Object-Manifest': dlo_prefix},
            return_resp=True)
        v2_version_id = resp.getheader('x-object-version-id')

        self.assertTotalVersions(self.env.container, 8)
        self.assertEqual(b'112233', man_file.read())

        self.assert_previous_version(
            obj_name, v1_version_id, b'old content', 'text/jibberish01')

        # overwrite the manifest with a normal file
        man_file.write(b'new content')
        self.assertTotalVersions(self.env.container, 9)
        self.assertEqual(b'new content', man_file.read())

        # new most-recent archive is the dlo
        self.assert_previous_version(
            obj_name, v2_version_id, b'112233', 'text/jibberish02',
            expected_headers={'X-Object-Manifest': dlo_prefix})
        return obj_name, man_file

    def test_versioning_dlo(self):
        obj_name, man_file = \
            self._test_versioning_dlo_setup()

        man_file.delete()
        with self.assertRaises(ResponseError) as cm:
            man_file.read()
        self.assertEqual(404, cm.exception.status)

        # 9 plus one more write
        self.assertTotalVersions(self.env.container, 10)

        expected = [b'old content', b'112233', b'new content']

        bodies = []
        listing_parms = {'versions': None, 'format': 'json',
                         'reverse': 'true', 'prefix': obj_name}
        for obj_info in self.env.container.files(parms=listing_parms)[:3]:
            bodies.append(man_file.read(
                parms={'version-id': obj_info['version_id']}))
        self.assertEqual(expected, bodies)

    def _check_overwriting_symlink(self):
        # sanity
        container = self.env.container
        self.assertTrue(
            config_true_value(self.env.container.info()['versions_enabled']))

        tgt_a_name = Utils.create_name()
        tgt_b_name = Utils.create_name()
        expected_count = 0

        tgt_a = container.file(tgt_a_name)
        tgt_a.write(b'aaaaa', hdrs={'Content-Type': 'text/jibberish01'})
        expected_count += 1

        tgt_b = container.file(tgt_b_name)
        tgt_b.write(b"bbbbb")
        expected_count += 1

        symlink_name = Utils.create_name()
        sym_tgt_header = quote(unquote('%s/%s' % (container.name, tgt_a_name)))
        sym_headers_a = {'X-Symlink-Target': sym_tgt_header}
        symlink = container.file(symlink_name)
        resp = symlink.write(b'', hdrs=sym_headers_a, return_resp=True)
        v1_version_id = resp.getheader('x-object-version-id')
        expected_count += 1
        self.assertEqual(b"aaaaa", symlink.read())

        sym_headers_b = {'X-Symlink-Target': '%s/%s' % (container.name,
                                                        tgt_b_name)}
        symlink.write(b"", hdrs=sym_headers_b)
        expected_count += 1
        self.assertEqual(b"bbbbb", symlink.read())

        self.assertTotalVersions(container, expected_count)
        self.assert_previous_version(
            symlink_name, v1_version_id, b'aaaaa', 'text/jibberish01')
        return symlink, tgt_a

    def test_overwriting_symlink(self):
        if 'symlink' not in cluster_info:
            raise SkipTest("Symlinks not enabled")

        symlink, target = self._check_overwriting_symlink()
        # test delete
        symlink.delete()
        with self.assertRaises(ResponseError) as cm:
            symlink.read()
        self.assertEqual(404, cm.exception.status)

    def _setup_symlink(self):
        tgt_name = 'target-' + Utils.create_name()
        target = self.env.container.file(tgt_name)
        target.write(b'target object data',
                     hdrs={'Content-Type': 'text/jibberish01'})
        symlink = self.env.container.file('symlink')
        resp = symlink.write(b'', hdrs={
            'Content-Type': 'application/symlink',
            'X-Symlink-Target': '%s/%s' % (
                self.env.container.name, target.name)},
            return_resp=True)
        symlink_version_id = resp.getheader('x-object-version-id')
        return symlink, symlink_version_id, target

    def _check_copy_destination_symlink(self):
        symlink, sym_version_id, target = self._setup_symlink()
        self.assertEqual(b'target object data', symlink.read())
        symlink.write(b'this is not a symlink')

        # target, symlink, and new 'not a symlink' overwritten by write
        self.assertTotalVersions(self.env.container, 3)
        self.assert_previous_version(
            symlink.name, sym_version_id,
            b'target object data', 'text/jibberish01')

        # the symlink is still a symlink
        prev_version = self.env.container.file(symlink.name)
        prev_version.initialize(parms={'version-id': sym_version_id})
        self.assertEqual('application/symlink',
                         prev_version.info(parms={
                             'version-id': sym_version_id,
                             'symlink': 'get'})['content_type'])
        prev_version.copy(self.env.container.name, symlink.name,
                          parms={'version-id': sym_version_id,
                                 'symlink': 'get'})
        self.assertEqual(b'target object data', symlink.read())
        self.assertTotalVersions(self.env.container, 4)

        return symlink, target

    def test_copy_destination_restore_symlink(self):
        if 'symlink' not in cluster_info:
            raise SkipTest("Symlinks not enabled")

        symlink, target = self._check_copy_destination_symlink()
        symlink.delete()
        with self.assertRaises(ResponseError) as cm:
            symlink.read()
        self.assertEqual(404, cm.exception.status)
        # symlink & target, plus overwrite and restore, then delete marker
        self.assertTotalVersions(self.env.container, 5)

    def test_versioned_staticlink(self):
        tgt_name = 'target-' + Utils.create_name()
        link_name = 'staticlink-' + Utils.create_name()
        target = self.env.container.file(tgt_name)
        staticlink = self.env.container.file(link_name)

        target_resp = target.write(b'target object data', hdrs={
            'Content-Type': 'text/jibberish01'}, return_resp=True)
        staticlink.write(b'', hdrs={
            'X-Symlink-Target': '%s/%s' % (
                self.env.container.name, target.name),
            'X-Symlink-Target-Etag': target_resp.getheader('etag'),
        }, cfg={'no_content_type': True})
        self.assertEqual(b'target object data', staticlink.read())

        listing_parms = {'format': 'json', 'versions': 'true'}
        prev_versions = self.env.container.files(parms=listing_parms)
        expected = [{
            'name': link_name,
            'bytes': 0,
            'content_type': 'text/jibberish01',
            'is_latest': True,
        }, {
            'name': tgt_name,
            'bytes': 18,
            'content_type': 'text/jibberish01',
            'is_latest': True,
        }]
        self.assertEqual(expected, [{
            k: i[k] for k in (
                'name', 'bytes', 'content_type', 'is_latest',
            )} for i in prev_versions])

        target_resp = target.write(b'updated target data', hdrs={
            'Content-Type': 'text/jibberish02'}, return_resp=True)
        with self.assertRaises(ResponseError) as caught:
            staticlink.read()
        self.assertEqual(409, caught.exception.status)
        staticlink.write(b'', hdrs={
            'X-Symlink-Target': '%s/%s' % (
                self.env.container.name, target.name),
            'X-Symlink-Target-Etag': target_resp.getheader('etag'),
        }, cfg={'no_content_type': True})
        self.assertEqual(b'updated target data', staticlink.read())

        listing_parms = {'format': 'json', 'versions': 'true'}
        prev_versions = self.env.container.files(parms=listing_parms)
        expected = [{
            'name': link_name,
            'bytes': 0,
            'content_type': 'text/jibberish02',
            'is_latest': True,
        }, {
            'name': link_name,
            'bytes': 0,
            'content_type': 'text/jibberish01',
            'is_latest': False,
        }, {
            'name': tgt_name,
            'bytes': 19,
            'content_type': 'text/jibberish02',
            'is_latest': True,
        }, {
            'name': tgt_name,
            'bytes': 18,
            'content_type': 'text/jibberish01',
            'is_latest': False,
        }]
        self.assertEqual(expected, [{
            k: i[k] for k in (
                'name', 'bytes', 'content_type', 'is_latest',
            )} for i in prev_versions])

    def test_link_to_versioned_object(self):

        # setup target object
        tgt_name = 'target-' + Utils.create_name()
        target = self.env.container.file(tgt_name)
        target_resp = target.write(b'target object data', hdrs={
            'Content-Type': 'text/jibberish01'}, return_resp=True)

        # setup dynamic link object from a non-versioned container
        link_container_name = 'link-container-' + Utils.create_name()
        link_name = 'link-' + Utils.create_name()
        link_cont = self.env.account.container(link_container_name)
        self.assertTrue(link_cont.create())
        link = link_cont.file(link_name)
        self.assertTrue(link.write(b'', hdrs={
            'X-Symlink-Target': '%s/%s' % (
                self.env.container.name, tgt_name),
        }, cfg={'no_content_type': True}))
        self.assertEqual(b'target object data', link.read())

        # setup static link object from a non-versioned container
        staticlink_name = 'staticlink-' + Utils.create_name()
        staticlink = link_cont.file(staticlink_name)
        self.assertTrue(staticlink.write(b'', hdrs={
            'X-Symlink-Target': '%s/%s' % (
                self.env.container.name, tgt_name),
            'X-Symlink-Target-Etag': target_resp.getheader('etag'),
        }, cfg={'no_content_type': True}))
        self.assertEqual(b'target object data', link.read())

    def test_versioned_post(self):
        # first we'll create a versioned object
        obj_name = Utils.create_name()
        obj = self.env.container.file(obj_name)
        resp = obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish10'
        }, return_resp=True)
        v1_version_id = resp.getheader('x-object-version-id')

        # send post request
        obj.post(hdrs={'Content-Type': 'text/updated20'})

        # head request should show updated content-type
        obj_info = obj.info()
        self.assertEqual(obj_info['content_type'], 'text/updated20')

        listing_parms = {'format': 'json', 'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(1, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj_name,
            'bytes': 8,
            'content_type': 'text/updated20',
            'hash': '966634ebf2fc135707d6753692bf4b1e',
            'version_id': v1_version_id,
            'is_latest': True,
        }])

    def test_unversioned_post(self):
        # first we'll create a versioned object
        obj_name = Utils.create_name()
        obj = self.env.container.file(obj_name)
        resp = obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish10'
        }, return_resp=True)
        v1_version_id = resp.getheader('x-object-version-id')

        # now, turn off versioning
        self.env.container.update_metadata(
            hdrs={self.env.versions_header_key: 'False'})

        obj.post(hdrs={'Content-Type': 'text/updated20'})

        # head request should show updated content-type
        obj_info = obj.info()
        self.assertEqual(obj_info['content_type'], 'text/updated20')

        listing_parms = {'format': 'json', 'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(1, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj_name,
            'bytes': 8,
            'content_type': 'text/updated20',
            'hash': '966634ebf2fc135707d6753692bf4b1e',
            'is_latest': True,
            'version_id': v1_version_id,
            'is_latest': True,
        }])

    def test_unversioned_overwrite_and_delete(self):
        # first we'll create a versioned object
        obj_name = Utils.create_name()
        obj = self.env.container.file(obj_name)
        resp = obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish18'
        }, return_resp=True)
        v1_version_id = resp.getheader('x-object-version-id')
        self.assertTotalVersions(self.env.container, 1)

        # now, turn off versioning, and delete source obj
        self.env.container.update_metadata(
            hdrs={self.env.versions_header_key: 'False'})
        obj.delete()

        # no delete markers, archive listing is unchanged
        self.assertTotalVersions(self.env.container, 1)

        # sanity, object is gone
        self.assertRaises(ResponseError, obj.read)
        self.assertEqual(404, obj.conn.response.status)

        # but, archive version is unmodified
        self.assert_previous_version(obj_name, v1_version_id, b'version1',
                                     'text/jibberish18')

        # a new overwrites will not have a version-id
        resp = obj.write(b'version2', hdrs={
            'Content-Type': 'text/jibberish19'
        }, return_resp=True)
        self.assertIsNone(resp.getheader('x-object-version-id'))
        self.assertTotalVersions(self.env.container, 2)

        resp = obj.write(b'version3', hdrs={
            'Content-Type': 'text/jibberish20'
        }, return_resp=True)
        self.assertIsNone(resp.getheader('x-object-version-id'))
        self.assertTotalVersions(self.env.container, 2)

        obj.delete()
        self.assertTotalVersions(self.env.container, 1)

        obj.delete(tolerate_missing=True)
        self.assertTotalVersions(self.env.container, 1)

    def test_versioned_overwrite_from_old_version(self):
        versioned_obj_name = Utils.create_name()
        obj = self.env.container.file(versioned_obj_name)
        resp = obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish32'
        }, return_resp=True)
        v1_version_id = resp.getheader('x-object-version-id')
        v1_etag = normalize_etag(resp.getheader('etag'))

        resp = obj.write(b'version2', hdrs={
            'Content-Type': 'text/jibberish33'
        }, return_resp=True)
        v2_version_id = resp.getheader('x-object-version-id')
        v2_etag = normalize_etag(resp.getheader('etag'))

        # sanity
        self.assertEqual(b'version2', obj.read())

        self.assertTotalVersions(self.env.container, 2)
        listing_parms = {'format': 'json', 'reverse': 'true', 'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(2, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': versioned_obj_name,
            'bytes': 8,
            'content_type': 'text/jibberish32',
            'hash': v1_etag,
            'version_id': v1_version_id,
            'is_latest': False,
        }, {
            'name': versioned_obj_name,
            'bytes': 8,
            'content_type': 'text/jibberish33',
            'hash': v2_etag,
            'version_id': v2_version_id,
            'is_latest': True,
        }])

        # restore old version1 back in place with a copy request
        # should get a new version-id
        old_version_obj = self.env.container.file(versioned_obj_name)
        resp = old_version_obj.copy(self.env.container.name,
                                    versioned_obj_name,
                                    parms={'version-id': v1_version_id},
                                    return_resp=True)
        v3_version_id = resp.getheader('x-object-version-id')

        listing_parms = {'format': 'json', 'reverse': 'true', 'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(3, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': versioned_obj_name,
            'bytes': 8,
            'content_type': 'text/jibberish32',
            'hash': v1_etag,
            'version_id': v1_version_id,
            'is_latest': False,
        }, {
            'name': versioned_obj_name,
            'bytes': 8,
            'content_type': 'text/jibberish33',
            'hash': v2_etag,
            'version_id': v2_version_id,
            'is_latest': False,
        }, {
            'name': versioned_obj_name,
            'bytes': 8,
            'content_type': 'text/jibberish32',
            'hash': v1_etag,
            'version_id': v3_version_id,
            'is_latest': True,
        }])

        self.assertEqual(b'version1', obj.read())
        obj_info = obj.info()
        self.assertEqual('text/jibberish32', obj_info['content_type'])
        self.assertEqual(v1_etag, normalize_etag(obj_info['etag']))

    def test_delete_with_version_api_old_object(self):
        versioned_obj_name = Utils.create_name()
        obj = self.env.container.file(versioned_obj_name)
        resp = obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish32'
        }, return_resp=True)
        v1_version_id = resp.getheader('x-object-version-id')

        obj.write(b'version2', hdrs={'Content-Type': 'text/jibberish33'})

        # sanity
        self.assertEqual(b'version2', obj.read())

        self.assertTotalVersions(self.env.container, 2)
        obj.delete(parms={'version-id': v1_version_id})

        self.assertEqual(b'version2', obj.read())
        self.assertTotalVersions(self.env.container, 1)

    def test_delete_with_version_api_current_object(self):
        versioned_obj_name = Utils.create_name()
        obj = self.env.container.file(versioned_obj_name)
        obj.write(b'version1', hdrs={'Content-Type': 'text/jibberish32'})

        resp = obj.write(b'version2', hdrs={
            'Content-Type': 'text/jibberish33'
        }, return_resp=True)
        v2_version_id = resp.getheader('x-object-version-id')

        # sanity
        self.assertEqual(b'version2', obj.read())

        self.assertTotalVersions(self.env.container, 2)
        obj.delete(parms={'version-id': v2_version_id})

        with self.assertRaises(ResponseError) as cm:
            obj.read()
        self.assertEqual(404, cm.exception.status)
        self.assertTotalVersions(self.env.container, 1)

    def test_delete_delete_marker_with_version_api(self):
        versioned_obj_name = Utils.create_name()
        obj = self.env.container.file(versioned_obj_name)
        obj.write(b'version1', hdrs={'Content-Type': 'text/jibberish32'})

        obj.delete()
        resp_headers = {
            h.lower(): v for h, v in obj.conn.response.getheaders()}
        self.assertIn('x-object-version-id', resp_headers)
        dm_version_id = resp_headers['x-object-version-id']

        # sanity
        with self.assertRaises(ResponseError) as cm:
            obj.info(parms={'version-id': dm_version_id})
        resp_headers = {
            h.lower(): v for h, v in cm.exception.headers}
        self.assertEqual(dm_version_id,
                         resp_headers['x-object-version-id'])
        self.assertEqual(DELETE_MARKER_CONTENT_TYPE,
                         resp_headers['content-type'])

        obj.delete(parms={'version-id': dm_version_id})
        resp_headers = {
            h.lower(): v for h, v in obj.conn.response.getheaders()}
        self.assertEqual(dm_version_id,
                         resp_headers['x-object-version-id'])

    def test_delete_with_version_api_last_object(self):
        versioned_obj_name = Utils.create_name()
        obj = self.env.container.file(versioned_obj_name)
        resp = obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish1'
        }, return_resp=True)
        v1_version_id = resp.getheader('x-object-version-id')

        # sanity
        self.assertEqual(b'version1', obj.read())
        self.assertTotalVersions(self.env.container, 1)

        # delete
        obj.delete(parms={'version-id': v1_version_id})

        with self.assertRaises(ResponseError) as cm:
            obj.read()
        self.assertEqual(404, cm.exception.status)
        self.assertTotalVersions(self.env.container, 0)

    def test_delete_with_version_api_null_version(self):
        versioned_obj_name = Utils.create_name()
        obj = self.env.container.file(versioned_obj_name)
        obj.write(b'version1', hdrs={'Content-Type': 'text/jibberish32'})
        obj.write(b'version2', hdrs={'Content-Type': 'text/jibberish33'})

        # sanity
        self.assertEqual(b'version2', obj.read())
        self.assertTotalVersions(self.env.container, 2)

        obj.delete(parms={'version-id': 'null'})
        with self.assertRaises(ResponseError) as caught:
            obj.read()
        self.assertEqual(404, caught.exception.status)

        # no versions removed
        self.assertTotalVersions(self.env.container, 2)

    def test_delete_with_version_api_old_object_disabled(self):
        versioned_obj_name = Utils.create_name()
        obj = self.env.container.file(versioned_obj_name)
        resp = obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish32'
        }, return_resp=True)
        v1_version_id = resp.getheader('x-object-version-id')

        obj.write(b'version2', hdrs={'Content-Type': 'text/jibberish33'})

        # disabled versioning
        self.env.container.update_metadata(
            hdrs={self.env.versions_header_key: 'False'})

        # sanity
        self.assertEqual(b'version2', obj.read())

        self.assertTotalVersions(self.env.container, 2)
        obj.delete(parms={'version-id': v1_version_id})

        self.assertEqual(b'version2', obj.read())
        self.assertTotalVersions(self.env.container, 1)

    def test_delete_with_version_api_current_object_disabled(self):
        versioned_obj_name = Utils.create_name()
        obj = self.env.container.file(versioned_obj_name)
        obj.write(b'version1', hdrs={'Content-Type': 'text/jibberish32'})

        resp = obj.write(b'version2', hdrs={
            'Content-Type': 'text/jibberish33'
        }, return_resp=True)
        v2_version_id = resp.getheader('x-object-version-id')

        # disabled versioning
        self.env.container.update_metadata(
            hdrs={self.env.versions_header_key: 'False'})

        # sanity
        self.assertEqual(b'version2', obj.read())

        self.assertTotalVersions(self.env.container, 2)
        obj.delete(parms={'version-id': v2_version_id})

        with self.assertRaises(ResponseError) as cm:
            obj.read()
        self.assertEqual(404, cm.exception.status)
        self.assertTotalVersions(self.env.container, 1)

    def test_delete_with_version_api_old_object_current_unversioned(self):
        versioned_obj_name = Utils.create_name()
        obj = self.env.container.file(versioned_obj_name)
        resp = obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish32'
        }, return_resp=True)
        v1_version_id = resp.getheader('x-object-version-id')

        # disabled versioning
        self.env.container.update_metadata(
            hdrs={self.env.versions_header_key: 'False'})

        # write unversioned object (i.e., version-id='null')
        obj.write(b'version2', hdrs={'Content-Type': 'text/jibberish33'})

        # sanity
        self.assertEqual(b'version2', obj.read())

        self.assertTotalVersions(self.env.container, 2)
        obj.delete(parms={'version-id': v1_version_id})

        self.assertEqual(b'version2', obj.read())
        self.assertTotalVersions(self.env.container, 1)


class TestObjectVersioningUTF8(Base2, TestObjectVersioning):
    pass


class TestContainerOperations(TestObjectVersioningBase):

    def _prep_object_versions(self):

        # object with multiple versions and currently deleted
        obj1_v1 = {}
        obj1_v1['name'] = 'c' + Utils.create_name()
        obj = self.env.container.file(obj1_v1['name'])

        # v1
        resp = obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish11',
            'ETag': md5(b'version1').hexdigest(),
        }, return_resp=True)
        obj1_v1['id'] = resp.getheader('x-object-version-id')

        # v2
        resp = obj.write(b'version2', hdrs={
            'Content-Type': 'text/jibberish12',
            'ETag': md5(b'version2').hexdigest(),
        }, return_resp=True)
        obj1_v2 = {}
        obj1_v2['name'] = obj1_v1['name']
        obj1_v2['id'] = resp.getheader('x-object-version-id')

        # v3
        resp = obj.write(b'version3', hdrs={
            'Content-Type': 'text/jibberish13',
            'ETag': md5(b'version3').hexdigest(),
        }, return_resp=True)
        obj1_v3 = {}
        obj1_v3['name'] = obj1_v1['name']
        obj1_v3['id'] = resp.getheader('x-object-version-id')

        with self.assertRaises(ResponseError) as cm:
            obj.write(b'version4', hdrs={
                'Content-Type': 'text/jibberish11',
                'ETag': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
            })
        self.assertEqual(422, cm.exception.status)

        # v4
        obj.delete()
        resp_headers = {
            h.lower(): v for h, v in obj.conn.response.getheaders()}
        obj1_v4 = {}
        obj1_v4['name'] = obj1_v1['name']
        obj1_v4['id'] = resp_headers.get('x-object-version-id')

        # object with just a single version
        obj2_v1 = {}
        obj2_v1['name'] = 'b' + Utils.create_name()
        obj = self.env.container.file(obj2_v1['name'])
        resp = obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish20',
            'ETag': '966634ebf2fc135707d6753692bf4b1e',
        }, return_resp=True)
        obj2_v1['id'] = resp.getheader('x-object-version-id')

        # object never existed, just a delete marker
        obj3_v1 = {}
        obj3_v1['name'] = 'a' + Utils.create_name()
        obj = self.env.container.file(obj3_v1['name'])
        obj.delete(tolerate_missing=True)
        self.assertEqual(obj.conn.response.status, 404)
        resp_headers = {
            h.lower(): v for h, v in obj.conn.response.getheaders()}
        obj3_v1['id'] = resp_headers.get('x-object-version-id')

        return (obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1)

    def _prep_unversioned_objects(self):
        objs = (
            'deleted' + Utils.create_name(),
            'in' + Utils.create_name(),
            'order' + Utils.create_name(),
        )

        # object with multiple writes and currently deleted
        obj = self.env.unversioned_container.file(objs[0])
        obj.write(b'data', hdrs={
            'Content-Type': 'text/jibberish11',
            'ETag': md5(b'data').hexdigest(),
        })
        obj.delete()

        obj = self.env.unversioned_container.file(objs[1])
        obj.write(b'first', hdrs={
            'Content-Type': 'text/blah-blah-blah',
            'ETag': md5(b'first').hexdigest(),
        })

        obj = self.env.unversioned_container.file(objs[2])
        obj.write(b'second', hdrs={
            'Content-Type': 'text/plain',
            'ETag': md5(b'second').hexdigest(),
        })
        return objs

    def test_list_all_versions(self):
        obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1 = \
            self._prep_object_versions()

        # list all versions in container
        listing_parms = {'format': 'json', 'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(6, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj3_v1['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj3_v1['id'],
        }, {
            'name': obj2_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish20',
            'hash': '966634ebf2fc135707d6753692bf4b1e',
            'is_latest': True,
            'version_id': obj2_v1['id'],
        }, {
            'name': obj1_v4['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj1_v4['id'],
        }, {
            'name': obj1_v3['name'],
            'bytes': 8,
            'content_type': 'text/jibberish13',
            'hash': md5(b'version3').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v3['id'],
        }, {
            'name': obj1_v2['name'],
            'bytes': 8,
            'content_type': 'text/jibberish12',
            'hash': md5(b'version2').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v2['id'],
        }, {
            'name': obj1_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish11',
            'hash': md5(b'version1').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v1['id'],
        }])

    def test_list_all_versions_reverse(self):
        obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1 = \
            self._prep_object_versions()

        # list all versions in container in reverse order
        listing_parms = {'format': 'json', 'reverse': 'true', 'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(6, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj1_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish11',
            'hash': md5(b'version1').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v1['id'],
        }, {
            'name': obj1_v2['name'],
            'bytes': 8,
            'content_type': 'text/jibberish12',
            'hash': md5(b'version2').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v2['id'],
        }, {
            'name': obj1_v3['name'],
            'bytes': 8,
            'content_type': 'text/jibberish13',
            'hash': md5(b'version3').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v3['id'],
        }, {
            'name': obj1_v4['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj1_v4['id'],
        }, {
            'name': obj2_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish20',
            'hash': '966634ebf2fc135707d6753692bf4b1e',
            'is_latest': True,
            'version_id': obj2_v1['id'],
        }, {
            'name': obj3_v1['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj3_v1['id'],
        }])

    def test_list_versions_prefix(self):

        obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1 = \
            self._prep_object_versions()

        # list all versions for a given object
        listing_parms = {'format': 'json',
                         'versions': None, 'prefix': obj1_v1['name']}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(4, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj1_v4['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj1_v4['id'],
        }, {
            'name': obj1_v3['name'],
            'bytes': 8,
            'content_type': 'text/jibberish13',
            'hash': md5(b'version3').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v3['id'],
        }, {
            'name': obj1_v2['name'],
            'bytes': 8,
            'content_type': 'text/jibberish12',
            'hash': md5(b'version2').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v2['id'],
        }, {
            'name': obj1_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish11',
            'hash': md5(b'version1').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v1['id'],
        }])

    def test_list_versions_prefix_reverse(self):

        obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1 = \
            self._prep_object_versions()

        # list all versions for a given object in reverse order
        listing_parms = {'format': 'json', 'reverse': 'true',
                         'versions': None, 'prefix': obj1_v1['name']}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(4, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj1_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish11',
            'hash': md5(b'version1').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v1['id'],
        }, {
            'name': obj1_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish12',
            'hash': md5(b'version2').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v2['id'],
        }, {
            'name': obj1_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish13',
            'hash': md5(b'version3').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v3['id'],
        }, {
            'name': obj1_v1['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj1_v4['id'],
        }])

    def test_list_limit(self):
        obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1 = \
            self._prep_object_versions()

        # list all versions in container
        listing_parms = {'format': 'json',
                         'limit': 3,
                         'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(3, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj3_v1['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj3_v1['id'],
        }, {
            'name': obj2_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish20',
            'hash': '966634ebf2fc135707d6753692bf4b1e',
            'is_latest': True,
            'version_id': obj2_v1['id'],
        }, {
            'name': obj1_v4['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj1_v4['id'],
        }])

    def test_list_limit_marker(self):
        obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1 = \
            self._prep_object_versions()

        # list all versions in container
        listing_parms = {'format': 'json',
                         'limit': 2,
                         'marker': obj2_v1['name'],
                         'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(2, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj1_v4['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj1_v4['id'],
        }, {
            'name': obj1_v3['name'],
            'bytes': 8,
            'content_type': 'text/jibberish13',
            'hash': md5(b'version3').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v3['id'],
        }])

    def test_list_version_marker(self):
        obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1 = \
            self._prep_object_versions()

        # list all versions starting with version_marker
        listing_parms = {'format': 'json',
                         'marker': obj1_v3['name'],
                         'version_marker': obj1_v3['id'],
                         'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(2, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj1_v2['name'],
            'bytes': 8,
            'content_type': 'text/jibberish12',
            'hash': md5(b'version2').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v2['id'],
        }, {
            'name': obj1_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish11',
            'hash': md5(b'version1').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v1['id'],
        }])

    def test_list_version_marker_reverse(self):
        obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1 = \
            self._prep_object_versions()

        # list all versions starting with version_marker in reverse order
        listing_parms = {'format': 'json',
                         'marker': obj1_v3['name'],
                         'version_marker': obj1_v3['id'],
                         'reverse': 'true',
                         'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(3, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj1_v4['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj1_v4['id'],
        }, {
            'name': obj2_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish20',
            'hash': '966634ebf2fc135707d6753692bf4b1e',
            'is_latest': True,
            'version_id': obj2_v1['id'],
        }, {
            'name': obj3_v1['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj3_v1['id'],
        }])

    def test_list_prefix_version_marker(self):
        obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1 = \
            self._prep_object_versions()

        # list versions with prefix starting with version_marker
        listing_parms = {'format': 'json',
                         'prefix': obj1_v3['name'],
                         'marker': obj1_v3['name'],
                         'version_marker': obj1_v3['id'],
                         'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(2, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj1_v2['name'],
            'bytes': 8,
            'content_type': 'text/jibberish12',
            'hash': md5(b'version2').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v2['id'],
        }, {
            'name': obj1_v1['name'],
            'bytes': 8,
            'content_type': 'text/jibberish11',
            'hash': md5(b'version1').hexdigest(),
            'is_latest': False,
            'version_id': obj1_v1['id'],
        }])

    def test_list_prefix_version_marker_reverse(self):
        obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1 = \
            self._prep_object_versions()

        # list versions with prefix starting with version_marker
        # in reverse order
        listing_parms = {'format': 'json',
                         'prefix': obj1_v3['name'],
                         'marker': obj1_v3['name'],
                         'version_marker': obj1_v3['id'],
                         'reverse': 'true',
                         'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(1, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj1_v4['name'],
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': True,
            'version_id': obj1_v4['id'],
        }])

    def test_unacceptable(self):
        def do_test(format):
            with self.assertRaises(ResponseError) as caught:
                self.env.container.files(parms={
                    'format': format, 'versions': None})
            self.assertEqual(caught.exception.status, 406)

        do_test('plain')
        do_test('xml')

        def do_test(accept):
            with self.assertRaises(ResponseError) as caught:
                self.env.container.files(hdrs={'Accept': accept},
                                         parms={'versions': None})
            self.assertEqual(caught.exception.status, 406)

        do_test('text/plain')
        do_test('text/xml')
        do_test('application/xml')
        do_test('foo/bar')

    def testFileListingLimitMarkerPrefix(self):
        cont = self.env.container

        files = ['apple', 'banana', 'cacao', 'date', 'elderberry']
        for f in files:
            file_item = cont.file(f)
            self.assertTrue(file_item.write_random())
            # immediately ovewrite
            self.assertTrue(file_item.write_random())
            time.sleep(.01)  # guarantee that the timestamp changes

        # sanity
        for i in range(len(files)):
            f = files[i]
            for j in range(1, len(files) - i):
                self.assertEqual(cont.files(parms={'limit': j, 'marker': f}),
                                 files[i + 1: i + j + 1])
            self.assertEqual(cont.files(parms={'marker': f}), files[i + 1:])
            self.assertEqual(cont.files(parms={'marker': f, 'prefix': f}), [])
            self.assertEqual(cont.files(parms={'prefix': f}), [f])

        # repeat items in files list
        versions = [f2 for f1 in files for f2 in (f1,) * 2]

        # now list versions too
        v = 0
        for i in range(len(files)):
            f = files[i]
            for j in range(1, len(files) - i):
                self.assertEqual(versions[i + v + 2: i + j + v + 2], [
                    item['name'] for item in cont.files(parms={
                        'limit': j, 'marker': f, 'versions': None})])
            self.assertEqual(versions[v + i + 2:], [
                item['name'] for item in cont.files(parms={
                    'marker': f, 'versions': None})])
            self.assertEqual(cont.files(parms={'marker': f, 'prefix': f,
                                               'versions': None}), [])
            self.assertEqual([f, f], [
                item['name'] for item in cont.files(parms={
                    'prefix': f, 'versions': None})])
            v = v + 1

    def testPrefixAndLimit(self):
        cont = self.env.container

        prefix_file_count = 10
        limit_count = 2
        prefixs = ['apple/', 'banana/', 'cacao/']
        prefix_files = {}

        for prefix in prefixs:
            prefix_files[prefix] = []

            for i in range(prefix_file_count):
                file_item = cont.file(prefix + Utils.create_name())
                self.assertTrue(file_item.write_random())
                self.assertTrue(file_item.write_random())
                prefix_files[prefix].append(file_item.name)
                time.sleep(.01)  # guarantee that the timestamp changes

        versions_prefix_files = {}
        for prefix in prefixs:
            versions_prefix_files[prefix] = [f2 for f1 in prefix_files[prefix]
                                             for f2 in (f1,) * 2]
        # sanity
        for format_type in [None, 'json', 'xml']:
            for prefix in prefixs:
                files = cont.files(parms={'prefix': prefix,
                                          'format': format_type})
                if isinstance(files[0], dict):
                    files = [x.get('name', x.get('subdir')) for x in files]
                self.assertEqual(files, sorted(prefix_files[prefix]))

        # list versions
        for format_type in [None, 'json']:
            for prefix in prefixs:
                files = cont.files(parms={'prefix': prefix,
                                          'versions': None,
                                          'format': format_type})
                if isinstance(files[0], dict):
                    files = [x.get('name', x.get('subdir')) for x in files]
                self.assertEqual(files, sorted(versions_prefix_files[prefix]))

        # list versions
        for format_type in [None, 'json']:
            for prefix in prefixs:
                files = cont.files(parms={'limit': limit_count,
                                          'versions': None,
                                          'prefix': prefix,
                                          'format': format_type})
                if isinstance(files[0], dict):
                    files = [x.get('name', x.get('subdir')) for x in files]
                self.assertEqual(len(files), limit_count)

                for file_item in files:
                    self.assertTrue(file_item.startswith(prefix))

    def testListDelimiter(self):
        cont = self.env.container

        delimiter = '-'
        files = ['test', delimiter.join(['test', 'bar']),
                 delimiter.join(['test', 'foo'])]
        for f in files:
            file_item = cont.file(f)
            self.assertTrue(file_item.write_random())

        # object with no current version, just a delete marker
        del_file = 'del-baz'
        obj = self.env.container.file(del_file)
        obj.delete(tolerate_missing=True)
        self.assertEqual(obj.conn.response.status, 404)

        # now, turn off versioning and write a un-versioned obj
        self.env.container.update_metadata(
            hdrs={self.env.versions_header_key: 'False'})

        # a new write will not have a version-id
        off_file = 'off-xyz'
        obj = self.env.container.file(off_file)
        resp = obj.write(b'unversioned', return_resp=True)
        self.assertIsNone(resp.getheader('x-object-version-id'))

        # sanity
        # list latest, delete marker should not show-up
        for format_type in [None, 'json', 'xml']:
            results = cont.files(parms={'format': format_type})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['off-xyz', 'test', 'test-bar',
                                       'test-foo'])

            results = cont.files(parms={'delimiter': delimiter,
                                        'format': format_type})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['off-', 'test', 'test-'])

            results = cont.files(parms={'delimiter': delimiter,
                                        'format': format_type,
                                        'reverse': 'yes'})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['test-', 'test', 'off-'])

        # list versions, we should see delete marker here
        for format_type in [None, 'json']:
            results = cont.files(parms={'versions': None,
                                        'format': format_type})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['del-baz', 'off-xyz', 'test',
                                       'test-bar', 'test-foo'])

            results = cont.files(parms={'delimiter': delimiter,
                                        'versions': None,
                                        'format': format_type})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['del-', 'off-', 'test', 'test-'])

            results = cont.files(parms={'delimiter': delimiter,
                                        'versions': None,
                                        'format': format_type,
                                        'reverse': 'yes'})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['test-', 'test', 'off-', 'del-'])

    def testListMultiCharDelimiter(self):
        cont = self.env.container

        delimiter = '-&'
        files = ['test', delimiter.join(['test', 'bar']),
                 delimiter.join(['test', 'foo'])]
        for f in files:
            file_item = cont.file(f)
            self.assertTrue(file_item.write_random())

        # object with no current version, just a delete marker
        del_file = 'del-&baz'
        obj = self.env.container.file(del_file)
        obj.delete(tolerate_missing=True)
        self.assertEqual(obj.conn.response.status, 404)

        # now, turn off versioning and write a un-versioned obj
        self.env.container.update_metadata(
            hdrs={self.env.versions_header_key: 'False'})

        # a new write will not have a version-id
        off_file = 'off-&xyz'
        obj = self.env.container.file(off_file)
        resp = obj.write(b'unversioned', return_resp=True)
        self.assertIsNone(resp.getheader('x-object-version-id'))

        # sanity
        # list latest, delete marker should not show-up
        for format_type in [None, 'json', 'xml']:
            results = cont.files(parms={'format': format_type})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['off-&xyz', 'test', 'test-&bar',
                                       'test-&foo'])

            results = cont.files(parms={'delimiter': delimiter,
                                        'format': format_type})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['off-&', 'test', 'test-&'])

            results = cont.files(parms={'delimiter': delimiter,
                                        'format': format_type,
                                        'reverse': 'yes'})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['test-&', 'test', 'off-&'])

        # list versions, we should see delete marker here
        for format_type in [None, 'json']:
            results = cont.files(parms={'versions': None,
                                        'format': format_type})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['del-&baz', 'off-&xyz', 'test',
                                       'test-&bar', 'test-&foo'])

            results = cont.files(parms={'delimiter': delimiter,
                                        'versions': None,
                                        'format': format_type})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['del-&', 'off-&', 'test', 'test-&'])

            results = cont.files(parms={'delimiter': delimiter,
                                        'versions': None,
                                        'format': format_type,
                                        'reverse': 'yes'})
            if isinstance(results[0], dict):
                results = [x.get('name', x.get('subdir')) for x in results]
            self.assertEqual(results, ['test-&', 'test', 'off-&', 'del-&'])

    def test_bytes_count(self):

        container = self.env.container

        # first store a non-versioned object
        # disable versioning
        container.update_metadata(
            hdrs={self.env.versions_header_key: 'False'})
        self.assertFalse(
            config_true_value(container.info()['versions_enabled']))

        obj = container.file(Utils.create_name())
        self.assertTrue(obj.write(b'not-versioned'))
        self.assertTotalVersions(container, 1)

        # enable versioning
        container.update_metadata(
            hdrs={self.env.versions_header_key: 'True'})
        self.assertTrue(
            config_true_value(container.info()['versions_enabled']))

        obj1_v1, obj1_v2, obj1_v3, obj1_v4, obj2_v1, obj3_v1 = \
            self._prep_object_versions()

        self.assertEqual(int(container.info()['bytes_used']), 32 + obj.size)
        self.assertEqual(int(container.info()['object_count']), 2)
        self.assertTotalVersions(container, 7)

    def test_container_quota_bytes(self):
        if 'container_quotas' not in tf.cluster_info:
            raise SkipTest('Container quotas not enabled')

        if tf.in_process:
            tf.skip_if_no_xattrs()

        container = self.env.container

        # write two versions of 5 bytes each
        obj = container.file(Utils.create_name())
        self.assertTrue(obj.write(b'aaaaa'))
        self.assertTrue(obj.write(b'bbbbb'))
        self.assertTotalVersions(container, 2)

        # set X-Container-Meta-Quota-Bytes is 10
        container.update_metadata(
            hdrs={'X-Container-Meta-Quota-Bytes': '10'})
        self.assertEqual(container.info()['container_quota_bytes'], '10')

        with self.assertRaises(ResponseError) as cm:
            obj.write(b'ccccc')
        self.assertEqual(413, cm.exception.status)

        # reset container quota
        container.update_metadata(
            hdrs={'X-Container-Meta-Quota-Bytes': ''})

    def test_list_unversioned_container(self):
        _obj1, obj2, obj3 = self._prep_unversioned_objects()
        # _obj1 got deleted, so won't show up at all
        item2 = {
            'name': obj2,
            'bytes': 5,
            'content_type': 'text/blah-blah-blah',
            'hash': md5(b'first').hexdigest(),
            'is_latest': True,
            'version_id': 'null',
        }
        item3 = {
            'name': obj3,
            'bytes': 6,
            'content_type': 'text/plain',
            'hash': md5(b'second').hexdigest(),
            'is_latest': True,
            'version_id': 'null',
        }

        # version-aware listing works for unversioned containers
        listing_parms = {'format': 'json',
                         'versions': None}
        listing = self.env.unversioned_container.files(parms=listing_parms)
        for item in listing:
            item.pop('last_modified')
        self.assertEqual(listing, [item2, item3])

        listing_parms = {'format': 'json',
                         'prefix': obj2[:2],
                         'versions': None}
        listing = self.env.unversioned_container.files(parms=listing_parms)
        for item in listing:
            item.pop('last_modified')
        self.assertEqual(listing, [item2])

        listing_parms = {'format': 'json',
                         'marker': obj2,
                         'versions': None}
        listing = self.env.unversioned_container.files(parms=listing_parms)
        for item in listing:
            item.pop('last_modified')
        self.assertEqual(listing, [item3])

        listing_parms = {'format': 'json',
                         'delimiter': 'er',
                         'versions': None}
        listing = self.env.unversioned_container.files(parms=listing_parms)
        for item in listing:
            if 'name' in item:
                item.pop('last_modified')
        self.assertEqual(listing, [item2, {'subdir': 'order'}])

        listing_parms = {'format': 'json',
                         'reverse': 'true',
                         'versions': None}
        listing = self.env.unversioned_container.files(parms=listing_parms)
        for item in listing:
            item.pop('last_modified')
        self.assertEqual(listing, [item3, item2])

    def test_is_latest(self):
        obj = self.env.container.file(Utils.create_name())

        # v1
        resp = obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish11',
            'ETag': md5(b'version1').hexdigest(),
        }, return_resp=True)
        obj_v1 = resp.getheader('x-object-version-id')

        # v2
        resp = obj.write(b'version2', hdrs={
            'Content-Type': 'text/jibberish12',
            'ETag': md5(b'version2').hexdigest(),
        }, return_resp=True)
        obj_v2 = resp.getheader('x-object-version-id')

        obj.delete()
        resp_headers = {
            h.lower(): v for h, v in obj.conn.response.getheaders()}
        obj_v3 = resp_headers.get('x-object-version-id')

        resp = obj.write(b'version4', hdrs={
            'Content-Type': 'text/jibberish14',
            'ETag': md5(b'version4').hexdigest(),
        }, return_resp=True)
        obj_v4 = resp.getheader('x-object-version-id')

        listing_parms = {'format': 'json', 'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(4, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj.name,
            'bytes': 8,
            'content_type': 'text/jibberish14',
            'hash': md5(b'version4').hexdigest(),
            'is_latest': True,
            'version_id': obj_v4,
        }, {
            'name': obj.name,
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': False,
            'version_id': obj_v3,
        }, {
            'name': obj.name,
            'bytes': 8,
            'content_type': 'text/jibberish12',
            'hash': md5(b'version2').hexdigest(),
            'is_latest': False,
            'version_id': obj_v2,
        }, {
            'name': obj.name,
            'bytes': 8,
            'content_type': 'text/jibberish11',
            'hash': md5(b'version1').hexdigest(),
            'is_latest': False,
            'version_id': obj_v1,
        }])

        self.env.container.update_metadata(
            hdrs={self.env.versions_header_key: 'False'})

        # v5 - non-versioned
        obj.write(b'version5', hdrs={
            'Content-Type': 'text/jibberish15',
            'ETag': md5(b'version5').hexdigest(),
        })

        listing_parms = {'format': 'json', 'versions': None}
        prev_versions = self.env.container.files(parms=listing_parms)
        self.assertEqual(5, len(prev_versions))
        for pv in prev_versions:
            pv.pop('last_modified')
        self.assertEqual(prev_versions, [{
            'name': obj.name,
            'bytes': 8,
            'content_type': 'text/jibberish15',
            'hash': md5(b'version5').hexdigest(),
            'is_latest': True,
            'version_id': 'null',
        }, {
            'name': obj.name,
            'bytes': 8,
            'content_type': 'text/jibberish14',
            'hash': md5(b'version4').hexdigest(),
            'is_latest': False,
            'version_id': obj_v4,
        }, {
            'name': obj.name,
            'bytes': 0,
            'content_type': 'application/x-deleted;swift_versions_deleted=1',
            'hash': MD5_OF_EMPTY_STRING,
            'is_latest': False,
            'version_id': obj_v3,
        }, {
            'name': obj.name,
            'bytes': 8,
            'content_type': 'text/jibberish12',
            'hash': md5(b'version2').hexdigest(),
            'is_latest': False,
            'version_id': obj_v2,
        }, {
            'name': obj.name,
            'bytes': 8,
            'content_type': 'text/jibberish11',
            'hash': md5(b'version1').hexdigest(),
            'is_latest': False,
            'version_id': obj_v1,
        }])


class TestContainerOperationsUTF8(Base2, TestContainerOperations):
    pass


class TestDeleteContainer(TestObjectVersioningBase):
    def tearDown(self):
        # do nothing since test will delete all data + container
        pass

    def test_delete_container(self):
        # sanity
        container = self.env.container
        self.assertTrue(
            config_true_value(container.info()['versions_enabled']))
        self.assertTotalVersions(container, 0)

        # write an object to be versioned
        obj = container.file(Utils.create_name)
        obj.write(b"foo")
        self.assertTotalVersions(container, 1)

        # delete object and attempt to delete container
        obj.delete()
        self.assertTotalVersions(container, 2)

        # expect failure because versioning is enabled and
        # old versions still exist
        self.assertFalse(container.delete())

        # disable it
        container.update_metadata(
            hdrs={self.env.versions_header_key: 'False'})
        self.assertFalse(
            config_true_value(container.info()['versions_enabled']))

        # expect failure because old versions still exist
        self.assertFalse(container.delete())

        # delete older versions
        self._tear_down_files(container)
        self.assertTotalVersions(container, 0)

        # and finally delete container
        self.assertTrue(container.delete())


class TestSloWithVersioning(TestObjectVersioningBase):

    def setUp(self):
        super(TestSloWithVersioning, self).setUp()

        if 'slo' not in cluster_info:
            raise SkipTest("SLO not enabled")
        if tf.in_process:
            tf.skip_if_no_xattrs()

        # create a container with versioning
        self.env.versions_header_key = 'X-Versions-Enabled'
        self.container = self.env.account.container(Utils.create_name())
        container_headers = {self.env.versions_header_key: 'True'}
        if not self.container.create(hdrs=container_headers):
            raise ResponseError(self.conn.response)

        self.segments_container = self.env.account.container(
            Utils.create_name())
        if not self.segments_container.create():
            raise ResponseError(self.conn.response)

        # create some segments
        self.seg_info = {}
        for letter, size in (('a', 1024 * 1024),
                             ('b', 1024 * 1024)):
            seg_name = letter
            file_item = self.segments_container.file(seg_name)
            file_item.write((letter * size).encode('ascii'))
            self.seg_info[seg_name] = {
                'size_bytes': size,
                'etag': file_item.md5,
                'path': '/%s/%s' % (self.segments_container.name, seg_name)}

    @property
    def account_name(self):
        if not self._account_name:
            self._account_name = self.env.account.conn.storage_path.rsplit(
                '/', 1)[-1]
        return self._account_name

    def _create_manifest(self, seg_name):
        # create a manifest in the versioning container
        file_item = self.container.file("my-slo-manifest")
        resp = file_item.write(
            json.dumps([self.seg_info[seg_name]]).encode('ascii'),
            parms={'multipart-manifest': 'put'},
            return_resp=True)
        version_id = resp.getheader('x-object-version-id')
        return file_item, version_id

    def _assert_is_manifest(self, file_item, seg_name, version_id=None):
        if version_id:
            read_params = {'multipart-manifest': 'get',
                           'version-id': version_id}
        else:
            read_params = {'multipart-manifest': 'get'}
        manifest_body = file_item.read(parms=read_params)
        resp_headers = {
            h.lower(): v for h, v in file_item.conn.response.getheaders()}
        self.assertIn('x-static-large-object', resp_headers)
        self.assertEqual('application/json; charset=utf-8',
                         file_item.content_type)
        try:
            manifest = json.loads(manifest_body)
        except ValueError:
            self.fail("GET with multipart-manifest=get got invalid json")

        self.assertEqual(1, len(manifest))
        key_map = {'etag': 'hash', 'size_bytes': 'bytes', 'path': 'name'}
        for k_client, k_slo in key_map.items():
            self.assertEqual(self.seg_info[seg_name][k_client],
                             manifest[0][k_slo])

    def _assert_is_object(self, file_item, seg_data, version_id=None):
        if version_id:
            file_contents = file_item.read(parms={'version-id': version_id})
        else:
            file_contents = file_item.read()
        self.assertEqual(1024 * 1024, len(file_contents))
        self.assertEqual(seg_data, file_contents[:1])
        self.assertEqual(seg_data, file_contents[-1:])

    def tearDown(self):
        self._tear_down_files(self.container)

    def test_slo_manifest_version(self):
        file_item, v1_version_id = self._create_manifest('a')
        # sanity check: read the manifest, then the large object
        self._assert_is_manifest(file_item, 'a')
        self._assert_is_object(file_item, b'a')

        # upload new manifest
        file_item, v2_version_id = self._create_manifest('b')
        # sanity check: read the manifest, then the large object
        self._assert_is_manifest(file_item, 'b')
        self._assert_is_object(file_item, b'b')

        # we wrote two versions
        self.assertTotalVersions(self.container, 2)

        # check the version 1 is still a manifest
        self._assert_is_manifest(file_item, 'a', v1_version_id)
        self._assert_is_object(file_item, b'a', v1_version_id)

        # listing looks good
        file_info = file_item.info()
        manifest_info = file_item.info(parms={'multipart-manifest': 'get'})
        obj_list = self.container.files(parms={'format': 'json'})
        for o in obj_list:
            o.pop('last_modified')
            # TODO: add symlink_path back in expected
            o.pop('symlink_path')
        expected = {
            'bytes': file_info['content_length'],
            'content_type': 'application/octet-stream',
            'hash': normalize_etag(manifest_info['etag']),
            'name': 'my-slo-manifest',
            'slo_etag': file_info['etag'],
            'version_symlink': True,
        }
        self.assertEqual([expected], obj_list)

        # delete the newest manifest
        file_item.delete()

        # expect to have 3 versions now, last one being a delete-marker
        self.assertTotalVersions(self.container, 3)

        # restore version 1
        file_item.copy(self.container.name, file_item.name,
                       parms={'multipart-manifest': 'get',
                              'version-id': v1_version_id})
        self.assertTotalVersions(self.container, 4)
        self._assert_is_manifest(file_item, 'a')
        self._assert_is_object(file_item, b'a')

        # versioned container listing still looks slo-like
        file_info = file_item.info()
        manifest_info = file_item.info(parms={'multipart-manifest': 'get'})
        obj_list = self.container.files(parms={'format': 'json'})
        for o in obj_list:
            o.pop('last_modified')
            # TODO: add symlink_path back in expected
            o.pop('symlink_path')
        expected = {
            'bytes': file_info['content_length'],
            'content_type': 'application/octet-stream',
            'hash': normalize_etag(manifest_info['etag']),
            'name': 'my-slo-manifest',
            'slo_etag': file_info['etag'],
            'version_symlink': True,
        }
        self.assertEqual([expected], obj_list)

        status = file_item.conn.make_request(
            'DELETE', file_item.path,
            hdrs={'Accept': 'application/json'},
            parms={'multipart-manifest': 'delete',
                   'version-id': v1_version_id})
        body = file_item.conn.response.read()
        self.assertEqual(status, 200, body)
        resp = json.loads(body)
        self.assertEqual(resp['Response Status'], '200 OK')
        self.assertEqual(resp['Errors'], [])
        self.assertEqual(resp['Number Deleted'], 2)

        self.assertTotalVersions(self.container, 3)
        # Since we included the ?multipart-manifest=delete, segments
        # got cleaned up and now the current version is busted
        with self.assertRaises(ResponseError) as caught:
            file_item.read()
        self.assertEqual(409, caught.exception.status)

    def test_links_to_slo(self):
        file_item, v1_version_id = self._create_manifest('a')
        slo_info = file_item.info()

        symlink_name = Utils.create_name()
        sym_tgt_header = quote(unquote('%s/%s' % (
            self.container.name, file_item.name)))
        symlink = self.container.file(symlink_name)

        # symlink to the slo
        sym_headers = {'X-Symlink-Target': sym_tgt_header}
        symlink.write(b'', hdrs=sym_headers)
        self.assertEqual(slo_info, symlink.info())

        # hardlink to the slo
        sym_headers['X-Symlink-Target-Etag'] = slo_info['x_manifest_etag']
        symlink.write(b'', hdrs=sym_headers)
        self.assertEqual(slo_info, symlink.info())


class TestVersionsLocationWithVersioning(TestObjectVersioningBase):

    # create a container with versioned writes
    location_header_key = 'X-Versions-Location'

    def setUp(self):
        super(TestVersionsLocationWithVersioning, self).setUp()

        if six.PY2:
            # avoid getting a prefix that stops halfway through an encoded
            # character
            prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
        else:
            prefix = Utils.create_name()[:10]

        self.versions_container = self.env.account.container(
            prefix + "-versions")
        if not self.versions_container.create():
            raise ResponseError(self.conn.response)

        self.container = self.env.account.container(prefix + "-objs")
        container_headers = {
            self.location_header_key: quote(self.versions_container.name)}
        if not self.container.create(hdrs=container_headers):
            raise ResponseError(self.conn.response)

    def _prep_object_versions(self):

        # object with multiple versions
        object_name = Utils.create_name()
        obj = self.container.file(object_name)

        # v1
        obj.write(b'version1', hdrs={
            'Content-Type': 'text/jibberish11',
            'ETag': md5(b'version1').hexdigest(),
        })

        # v2
        obj.write(b'version2', hdrs={
            'Content-Type': 'text/jibberish12',
            'ETag': md5(b'version2').hexdigest(),
        })

        # v3
        obj.write(b'version3', hdrs={
            'Content-Type': 'text/jibberish13',
            'ETag': md5(b'version3').hexdigest(),
        })

        return obj

    def test_list_with_versions_param(self):
        obj = self._prep_object_versions()
        obj_name = obj.name

        listing_parms = {'format': 'json', 'versions': None}
        current_versions = self.container.files(parms=listing_parms)
        self.assertEqual(1, len(current_versions))
        for pv in current_versions:
            pv.pop('last_modified')
        self.assertEqual(current_versions, [{
            'name': obj_name,
            'bytes': 8,
            'content_type': 'text/jibberish13',
            'hash': md5(b'version3').hexdigest(),
            'is_latest': True,
            'version_id': 'null'
        }])

        prev_versions = self.versions_container.files(parms=listing_parms)
        self.assertEqual(2, len(prev_versions))

        for pv in prev_versions:
            pv.pop('last_modified')
            name = pv.pop('name')
            self.assertTrue(name.startswith('%03x%s/' % (len(obj_name),
                                                         obj_name)))

        self.assertEqual(prev_versions, [{
            'bytes': 8,
            'content_type': 'text/jibberish11',
            'hash': md5(b'version1').hexdigest(),
            'is_latest': True,
            'version_id': 'null',
        }, {
            'bytes': 8,
            'content_type': 'text/jibberish12',
            'hash': md5(b'version2').hexdigest(),
            'is_latest': True,
            'version_id': 'null'
        }])

    def test_delete_with_null_version_id(self):
        obj = self._prep_object_versions()

        # sanity
        self.assertEqual(b'version3', obj.read())

        obj.delete(parms={'version-id': 'null'})
        if self.location_header_key == 'X-Versions-Location':
            self.assertEqual(b'version2', obj.read())
        else:
            with self.assertRaises(ResponseError) as caught:
                obj.read()
            self.assertEqual(404, caught.exception.status)


class TestHistoryLocationWithVersioning(TestVersionsLocationWithVersioning):

    # create a container with versioned writes
    location_header_key = 'X-History-Location'


class TestVersioningAccountTempurl(TestObjectVersioningBase):
    env = TestTempurlEnv
    digest_name = 'sha1'

    def setUp(self):
        self.env.versions_header_key = 'X-Versions-Enabled'
        super(TestVersioningAccountTempurl, self).setUp()
        if self.env.tempurl_enabled is False:
            raise SkipTest("TempURL not enabled")
        elif self.env.tempurl_enabled is not True:
            # just some sanity checking
            raise Exception(
                "Expected tempurl_enabled to be True/False, got %r" %
                (self.env.tempurl_enabled,))

        if self.digest_name not in cluster_info['tempurl'].get(
                'allowed_digests', ['sha1']):
            raise SkipTest("tempurl does not support %s signatures" %
                           self.digest_name)

        self.digest = getattr(hashlib, self.digest_name)
        self.expires = int(time.time()) + 86400
        self.obj_tempurl_parms = self.tempurl_parms(
            'GET', self.expires, self.env.conn.make_path(self.env.obj.path),
            self.env.tempurl_key)

    def tempurl_parms(self, method, expires, path, key):
        path = unquote(path)
        if not six.PY2:
            method = method.encode('utf8')
            path = path.encode('utf8')
            key = key.encode('utf8')
        sig = hmac.new(
            key,
            b'%s\n%d\n%s' % (method, expires, path),
            self.digest).hexdigest()
        return {'temp_url_sig': sig, 'temp_url_expires': str(expires)}

    def test_PUT(self):
        obj = self.env.obj

        # give out a signature which allows a PUT to obj
        expires = int(time.time()) + 86400
        put_parms = self.tempurl_parms(
            'PUT', expires, self.env.conn.make_path(obj.path),
            self.env.tempurl_key)

        # try to overwrite existing object
        resp = obj.write(b"version2", parms=put_parms,
                         cfg={'no_auth_token': True},
                         return_resp=True)
        resp_headers = {
            h.lower(): v for h, v in resp.getheaders()}
        self.assertIn('x-object-version-id', resp_headers)

    def test_GET_latest(self):
        obj = self.env.obj

        expires = int(time.time()) + 86400
        get_parms = self.tempurl_parms(
            'GET', expires, self.env.conn.make_path(obj.path),
            self.env.tempurl_key)

        # get v1 object (., version-id=null, no symlinks involved)
        contents = obj.read(parms=get_parms, cfg={'no_auth_token': True})
        self.assert_status([200])
        self.assertEqual(contents, b"obj contents")

        # give out a signature which allows a PUT to obj
        expires = int(time.time()) + 86400
        put_parms = self.tempurl_parms(
            'PUT', expires, self.env.conn.make_path(obj.path),
            self.env.tempurl_key)

        # try to overwrite existing object
        resp = obj.write(b"version2", parms=put_parms,
                         cfg={'no_auth_token': True},
                         return_resp=True)
        resp_headers = {
            h.lower(): v for h, v in resp.getheaders()}
        self.assertIn('x-object-version-id', resp_headers)

        # get v2 object
        contents = obj.read(parms=get_parms, cfg={'no_auth_token': True})
        self.assert_status([200])
        self.assertEqual(contents, b"version2")

    def test_GET_version_id(self):
        # N.B.: The test is not intended to imply the desired behavior
        # of a tempurl GET with version-id. Currently version-id is simply
        # ignored as the latest version is always returned. In the future,
        # users should be able to create a tempurl with version-id as a
        # parameter.

        # overwrite object a couple more times
        obj = self.env.obj
        resp = obj.write(b"version2", return_resp=True)
        v2_version_id = resp.getheader('x-object-version-id')
        obj.write(b"version3!!!")

        expires = int(time.time()) + 86400
        get_parms = self.tempurl_parms(
            'GET', expires, self.env.conn.make_path(obj.path),
            self.env.tempurl_key)
        get_parms['version-id'] = v2_version_id

        contents = obj.read(parms=get_parms, cfg={'no_auth_token': True})
        self.assert_status([200])
        self.assertEqual(contents, b"version3!!!")


class TestVersioningContainerTempurl(TestObjectVersioningBase):
    env = TestContainerTempurlEnv
    digest_name = 'sha1'

    def setUp(self):
        self.env.versions_header_key = 'X-Versions-Enabled'
        super(TestVersioningContainerTempurl, self).setUp()
        if self.env.tempurl_enabled is False:
            raise SkipTest("TempURL not enabled")
        elif self.env.tempurl_enabled is not True:
            # just some sanity checking
            raise Exception(
                "Expected tempurl_enabled to be True/False, got %r" %
                (self.env.tempurl_enabled,))

        if self.digest_name not in cluster_info['tempurl'].get(
                'allowed_digests', ['sha1']):
            raise SkipTest("tempurl does not support %s signatures" %
                           self.digest_name)

        self.digest = getattr(hashlib, self.digest_name)
        expires = int(time.time()) + 86400
        sig = self.tempurl_sig(
            'GET', expires, self.env.conn.make_path(self.env.obj.path),
            self.env.tempurl_key)
        self.obj_tempurl_parms = {'temp_url_sig': sig,
                                  'temp_url_expires': str(expires)}

    def tempurl_sig(self, method, expires, path, key):
        path = unquote(path)
        if not six.PY2:
            method = method.encode('utf8')
            path = path.encode('utf8')
            key = key.encode('utf8')
        return hmac.new(
            key,
            b'%s\n%d\n%s' % (method, expires, path),
            self.digest).hexdigest()

    def test_PUT(self):
        obj = self.env.obj

        # give out a signature which allows a PUT to new_obj
        expires = int(time.time()) + 86400
        sig = self.tempurl_sig(
            'PUT', expires, self.env.conn.make_path(obj.path),
            self.env.tempurl_key)
        put_parms = {'temp_url_sig': sig,
                     'temp_url_expires': str(expires)}

        # try to overwrite existing object
        resp = obj.write(b"version2", parms=put_parms,
                         cfg={'no_auth_token': True},
                         return_resp=True)
        resp_headers = {
            h.lower(): v for h, v in resp.getheaders()}
        self.assertIn('x-object-version-id', resp_headers)

    def test_GET_latest(self):
        obj = self.env.obj

        expires = int(time.time()) + 86400
        sig = self.tempurl_sig(
            'GET', expires, self.env.conn.make_path(obj.path),
            self.env.tempurl_key)
        get_parms = {'temp_url_sig': sig,
                     'temp_url_expires': str(expires)}

        # get v1 object (., version-id=null, no symlinks involved)
        contents = obj.read(parms=get_parms, cfg={'no_auth_token': True})
        self.assert_status([200])
        self.assertEqual(contents, b"obj contents")

        # overwrite existing object
        obj.write(b"version2")

        # get v2 object (reading from versions container)
        # versioning symlink allows us to bypass the normal
        # container-tempurl-key scoping
        contents = obj.read(parms=get_parms, cfg={'no_auth_token': True})
        self.assert_status([200])
        self.assertEqual(contents, b"version2")
        # HEAD works, too
        obj.info(parms=get_parms, cfg={'no_auth_token': True})
        self.assert_status([200])