# test_bitmap.py -- Compatibility tests for git pack bitmaps. # Copyright (C) 2025 Jelmer Vernooij # # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU # General Public License as published by the Free Software Foundation; version 2.0 # or (at your option) any later version. You can redistribute it and/or # modify it under the terms of either of these two licenses. # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # You should have received a copy of the licenses; if not, see # for a copy of the GNU General Public License # and for a copy of the Apache # License, Version 2.0. # """Compatibility tests for git pack bitmaps.""" import os import tempfile from dulwich.bitmap import ( BITMAP_OPT_FULL_DAG, BITMAP_OPT_HASH_CACHE, BITMAP_OPT_LOOKUP_TABLE, BitmapEntry, EWAHBitmap, PackBitmap, write_bitmap, ) from dulwich.object_format import DEFAULT_OBJECT_FORMAT from dulwich.pack import Pack from dulwich.repo import Repo from .. import TestCase from .utils import remove_ro, require_git_version, rmtree_ro, run_git_or_fail class BitmapCompatTests(TestCase): """Compatibility tests for reading git-generated bitmaps.""" def setUp(self): super().setUp() # Git bitmap support was added in 2.0.0 require_git_version((2, 0, 0)) self._tempdir = tempfile.mkdtemp() self.addCleanup(rmtree_ro, self._tempdir) def _init_repo_with_bitmap(self): """Create a repo and generate a bitmap using git.""" repo_path = os.path.join(self._tempdir, "test-repo") os.mkdir(repo_path) # Initialize repo run_git_or_fail(["init"], cwd=repo_path) # Create some commits test_file = os.path.join(repo_path, "test.txt") for i in range(5): with open(test_file, "w") as f: f.write(f"Content {i}\n") run_git_or_fail(["add", "test.txt"], cwd=repo_path) run_git_or_fail( ["commit", "-m", f"Commit {i}"], cwd=repo_path, env={"GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.com"}, ) # Enable bitmap writing and repack run_git_or_fail( ["config", "pack.writeBitmaps", "true"], cwd=repo_path, ) run_git_or_fail(["repack", "-a", "-d", "-b"], cwd=repo_path) return repo_path def test_read_git_generated_bitmap(self): """Test that Dulwich can read a bitmap generated by git.""" repo_path = self._init_repo_with_bitmap() # Find the pack file with bitmap pack_dir = os.path.join(repo_path, ".git", "objects", "pack") bitmap_files = [f for f in os.listdir(pack_dir) if f.endswith(".bitmap")] # Get the pack file (basename without extension) bitmap_name = bitmap_files[0] pack_basename = bitmap_name.replace(".bitmap", "") pack_path = os.path.join(pack_dir, pack_basename) # Verify bitmap file exists at expected location bitmap_path = pack_path + ".bitmap" self.assertTrue( os.path.exists(bitmap_path), f"Bitmap file not found at {bitmap_path}" ) # Try to load the bitmap using Dulwich with Pack(pack_path, object_format=DEFAULT_OBJECT_FORMAT) as pack: bitmap = pack.bitmap # Basic checks self.assertIsNotNone(bitmap, f"Failed to load bitmap from {pack_path}") self.assertIsNotNone(bitmap.pack_checksum, "Bitmap missing pack checksum") # Check that we have some type bitmaps # At minimum, we should have some commits self.assertGreater( len(bitmap.commit_bitmap.bits), 0, "Commit bitmap should not be empty", ) def test_git_can_use_dulwich_repo_with_bitmap(self): """Test that git can work with a repo that has Dulwich-created objects.""" repo_path = os.path.join(self._tempdir, "dulwich-repo") # Create a repo with Dulwich and add commits to ensure git creates bitmaps repo = Repo.init(repo_path, mkdir=True) self.addCleanup(repo.close) # Create actual commits, not just loose objects - git needs commits for bitmaps test_file = os.path.join(repo_path, "test.txt") for i in range(5): with open(test_file, "w") as f: f.write(f"Content {i}\n") run_git_or_fail(["add", "test.txt"], cwd=repo_path) run_git_or_fail( ["commit", "-m", f"Commit {i}"], cwd=repo_path, env={"GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.com"}, ) # Configure git to write bitmaps run_git_or_fail( ["config", "pack.writeBitmaps", "true"], cwd=repo_path, ) # Git should be able to repack with bitmaps run_git_or_fail(["repack", "-a", "-d", "-b"], cwd=repo_path) # Verify git created a bitmap pack_dir = os.path.join(repo_path, ".git", "objects", "pack") self.assertTrue(os.path.exists(pack_dir), "Pack directory should exist") bitmap_files = [f for f in os.listdir(pack_dir) if f.endswith(".bitmap")] self.assertGreater( len(bitmap_files), 0, "Git should have created a bitmap file after repack" ) def test_git_can_read_dulwich_bitmap(self): """Test that git can read a bitmap file written by Dulwich.""" repo_path = os.path.join(self._tempdir, "dulwich-bitmap-repo") # Create a repo with git and add commits run_git_or_fail(["init"], cwd=None, env={"GIT_DIR": repo_path}) test_file = os.path.join(repo_path, "..", "test.txt") os.makedirs(os.path.dirname(test_file), exist_ok=True) for i in range(5): with open(test_file, "w") as f: f.write(f"Content {i}\n") run_git_or_fail( ["add", test_file], cwd=os.path.dirname(repo_path), env={ "GIT_DIR": repo_path, "GIT_WORK_TREE": os.path.dirname(repo_path), }, ) run_git_or_fail( ["commit", "-m", f"Commit {i}"], cwd=os.path.dirname(repo_path), env={ "GIT_DIR": repo_path, "GIT_WORK_TREE": os.path.dirname(repo_path), "GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.com", }, ) # Create a pack with git first run_git_or_fail(["repack", "-a", "-d"], cwd=None, env={"GIT_DIR": repo_path}) # Now use Dulwich to write a bitmap for the pack pack_dir = os.path.join(repo_path, "objects", "pack") pack_files = [f for f in os.listdir(pack_dir) if f.endswith(".pack")] self.assertGreater(len(pack_files), 0, "Should have at least one pack file") pack_basename = pack_files[0].replace(".pack", "") pack_path = os.path.join(pack_dir, pack_basename) # Load the pack and create bitmap data, then close before writing with Pack(pack_path, object_format=DEFAULT_OBJECT_FORMAT) as pack: # Create a simple bitmap for testing # Git requires BITMAP_OPT_FULL_DAG flag bitmap = PackBitmap( flags=BITMAP_OPT_FULL_DAG | BITMAP_OPT_HASH_CACHE | BITMAP_OPT_LOOKUP_TABLE ) bitmap.pack_checksum = pack.get_stored_checksum() # Add bitmap entries for the first few commits in the pack for i, (sha, offset, crc) in enumerate(pack.index.iterentries()): if i >= 3: # Just add 3 entries break ewah = EWAHBitmap() # Mark this object and a couple others as reachable for j in range(i + 1): ewah.add(j) entry = BitmapEntry(object_pos=i, xor_offset=0, flags=0, bitmap=ewah) bitmap.entries[sha] = entry bitmap.entries_list.append((sha, entry)) # Add name hash cache bitmap.name_hash_cache = [0x12345678, 0xABCDEF00, 0xFEDCBA98] # Write the bitmap after pack is closed to avoid file locking on Windows bitmap_path = pack_path + ".bitmap" remove_ro(bitmap_path) write_bitmap(bitmap_path, bitmap) # Verify git can use the repository with our bitmap # This should succeed if git can read our bitmap run_git_or_fail( ["rev-list", "--count", "--use-bitmap-index", "HEAD"], cwd=None, env={"GIT_DIR": repo_path}, ) # Verify git count-objects works with our bitmap run_git_or_fail(["count-objects", "-v"], cwd=None, env={"GIT_DIR": repo_path}) def test_bitmap_file_format_structure(self): """Test that git-generated bitmap has expected structure.""" repo_path = self._init_repo_with_bitmap() # Find bitmap pack_dir = os.path.join(repo_path, ".git", "objects", "pack") bitmap_files = [f for f in os.listdir(pack_dir) if f.endswith(".bitmap")] bitmap_path = os.path.join(pack_dir, bitmap_files[0]) # Read the raw file to verify header with open(bitmap_path, "rb") as f: signature = f.read(4) self.assertEqual(b"BITM", signature, "Invalid bitmap signature") version = int.from_bytes(f.read(2), byteorder="big") self.assertGreaterEqual(version, 1, "Bitmap version should be >= 1") # Load with Dulwich and verify structure bitmap_name = bitmap_files[0] pack_basename = bitmap_name.replace(".bitmap", "") pack_path = os.path.join(pack_dir, pack_basename) with Pack(pack_path, object_format=DEFAULT_OBJECT_FORMAT) as pack: bitmap = pack.bitmap self.assertIsNotNone(bitmap) self.assertEqual(bitmap.version, version)