-
-
Notifications
You must be signed in to change notification settings - Fork 385
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add support for PackBuilder methods #1048
Changes from 5 commits
9c1ea03
9baf405
c415819
402fa4e
127c2c9
ecdafb8
902b6be
1b16f76
8b8a0b5
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,80 @@ | ||
# Copyright 2010-2020 The pygit2 contributors | ||
# | ||
# This file is free software; you can redistribute it and/or modify | ||
# it under the terms of the GNU General Public License, version 2, | ||
# as published by the Free Software Foundation. | ||
# | ||
# In addition to the permissions in the GNU General Public License, | ||
# the authors give you unlimited permission to link the compiled | ||
# version of this file into combinations with other programs, | ||
# and to distribute those combinations without any restriction | ||
# coming from the use of this file. (The General Public License | ||
# restrictions do apply in other respects; for example, they cover | ||
# modification of the file, and distribution when not linked into | ||
# a combined executable.) | ||
# | ||
# This file is distributed in the hope that it will be useful, but | ||
# WITHOUT ANY WARRANTY; without even the implied warranty of | ||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
# General Public License for more details. | ||
# | ||
# You should have received a copy of the GNU General Public License | ||
# along with this program; see the file COPYING. If not, write to | ||
# the Free Software Foundation, 51 Franklin Street, Fifth Floor, | ||
# Boston, MA 02110-1301, USA. | ||
|
||
|
||
# Import from pygit2 | ||
from .errors import check_error | ||
from .ffi import ffi, C | ||
from .utils import to_bytes | ||
|
||
|
||
class PackBuilder: | ||
|
||
def __init__(self, repo): | ||
|
||
cpackbuilder = ffi.new('git_packbuilder **') | ||
err = C.git_packbuilder_new(cpackbuilder, repo._repo) | ||
check_error(err) | ||
|
||
self._repo = repo | ||
self._packbuilder = cpackbuilder[0] | ||
self._cpackbuilder = cpackbuilder | ||
|
||
@property | ||
def _pointer(self): | ||
return bytes(ffi.buffer(self._packbuilder)[:]) | ||
|
||
def __del__(self): | ||
C.git_packbuilder_free(self._packbuilder) | ||
|
||
def __len__(self): | ||
return C.git_packbuilder_object_count(self._packbuilder) | ||
|
||
@staticmethod | ||
def convert_object_to_oid(git_object): | ||
oid = ffi.new('git_oid *') | ||
ffi.buffer(oid)[:] = git_object.raw[:] | ||
return oid | ||
|
||
def add(self, git_object): | ||
oid = self.convert_object_to_oid(git_object) | ||
err = C.git_packbuilder_insert(self._packbuilder, oid, ffi.NULL) | ||
check_error(err) | ||
|
||
def add_recur(self, git_object): | ||
oid = self.convert_object_to_oid(git_object) | ||
err = C.git_packbuilder_insert_recur(self._packbuilder, oid, ffi.NULL) | ||
check_error(err) | ||
|
||
def set_max_threads(self, n_threads): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Docs for It doesn't say anything about maximum, so please rename this method to |
||
return C.git_packbuilder_set_threads(self._packbuilder, n_threads) | ||
|
||
def write(self, path): | ||
err = C.git_packbuilder_write(self._packbuilder, to_bytes(path), 0, ffi.NULL, ffi.NULL) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Have updated to support NULL |
||
check_error(err) | ||
|
||
@property | ||
def written_objects_count(self): | ||
return C.git_packbuilder_written(self._packbuilder) |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -49,6 +49,7 @@ | |
from .blame import Blame | ||
from .utils import to_bytes, StrArray | ||
from .submodule import Submodule | ||
from .packbuilder import PackBuilder | ||
|
||
|
||
class BaseRepository(_Repository): | ||
|
@@ -83,6 +84,39 @@ def write(self, *args, **kwargs): | |
object.""" | ||
return self.odb.write(*args, **kwargs) | ||
|
||
def pack(self, path, pack_delegate=None, max_threads=None): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same as above, |
||
"""Pack the objects in the odb chosen by the pack_delegate function | ||
and write .pack and .idx files for them. | ||
|
||
Returns: the number of objects written to the pack | ||
|
||
Parameters: | ||
|
||
path | ||
The path to which the .pack and .idx files should be written. | ||
|
||
pack_delegate | ||
The method which will provide add the objects to the pack builder. Defaults to all objects. | ||
|
||
max_threads | ||
The maximum number of threads the PackBuilder will spawn. | ||
""" | ||
|
||
def pack_all_objects(pack_builder): | ||
for obj in self.odb: | ||
pack_builder.add(obj) | ||
|
||
pack_delegate = pack_delegate or pack_all_objects | ||
|
||
builder = PackBuilder(self) | ||
if max_threads: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. libgit2 docs say: when set to 0, libgit2 will autodetect the number of CPUs But here if the user passes zero
Please update the documentation string as well. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. have updated to support 0 and renamed the method and argument |
||
builder.set_max_threads(max_threads) | ||
pack_delegate(builder) | ||
builder.write(path) | ||
|
||
return builder.written_objects_count | ||
|
||
|
||
def __iter__(self): | ||
return iter(self.odb) | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,112 @@ | ||
# Copyright 2010-2020 The pygit2 contributors | ||
# | ||
# This file is free software; you can redistribute it and/or modify | ||
# it under the terms of the GNU General Public License, version 2, | ||
# as published by the Free Software Foundation. | ||
# | ||
# In addition to the permissions in the GNU General Public License, | ||
# the authors give you unlimited permission to link the compiled | ||
# version of this file into combinations with other programs, | ||
# and to distribute those combinations without any restriction | ||
# coming from the use of this file. (The General Public License | ||
# restrictions do apply in other respects; for example, they cover | ||
# modification of the file, and distribution when not linked into | ||
# a combined executable.) | ||
# | ||
# This file is distributed in the hope that it will be useful, but | ||
# WITHOUT ANY WARRANTY; without even the implied warranty of | ||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
# General Public License for more details. | ||
# | ||
# You should have received a copy of the GNU General Public License | ||
# along with this program; see the file COPYING. If not, write to | ||
# the Free Software Foundation, 51 Franklin Street, Fifth Floor, | ||
# Boston, MA 02110-1301, USA. | ||
|
||
"""Tests for Index files.""" | ||
|
||
import os | ||
|
||
import pytest | ||
|
||
import pygit2 | ||
from pygit2 import PackBuilder | ||
from . import utils | ||
from .utils import rmtree | ||
|
||
|
||
def test_create_packbuilder(testrepo): | ||
# simple test of PackBuilder creation | ||
packbuilder = PackBuilder(testrepo) | ||
assert len(packbuilder) == 0 | ||
|
||
|
||
def test_add(testrepo): | ||
# Add a few objects and confirm that the count is correct | ||
packbuilder = PackBuilder(testrepo) | ||
objects_to_add = [obj for obj in testrepo] | ||
packbuilder.add(objects_to_add[0]) | ||
assert len(packbuilder) == 1 | ||
packbuilder.add(objects_to_add[1]) | ||
assert len(packbuilder) == 2 | ||
|
||
|
||
def test_add_recursively(testrepo): | ||
# Add the head object and referenced objects recursively and confirm that the count is correct | ||
packbuilder = PackBuilder(testrepo) | ||
packbuilder.add_recur(testrepo.head.target) | ||
|
||
#expect a count of 4 made up of the following referenced objects: | ||
# Commit | ||
# Tree | ||
# Blob: hello.txt | ||
# Blob: .gitignore | ||
|
||
assert len(packbuilder) == 4 | ||
|
||
|
||
def test_repo_pack(testrepo, tmp_path): | ||
# pack the repo with the default strategy | ||
confirm_same_repo_after_packing(testrepo, tmp_path, None) | ||
|
||
|
||
def test_pack_with_delegate(testrepo, tmp_path): | ||
# loop through all branches and add each commit to the packbuilder | ||
def pack_delegate(pb): | ||
for branch in pb._repo.branches: | ||
br = pb._repo.branches.get(branch) | ||
for commit in br.log(): | ||
pb.add_recur(commit.oid_new) | ||
confirm_same_repo_after_packing(testrepo, tmp_path, pack_delegate) | ||
|
||
|
||
def setup_second_repo(tmp_path): | ||
# helper method to set up a second repo for comparison | ||
tmp_path_2 = os.path.join(tmp_path, 'test_repo2') | ||
with utils.TemporaryRepository('testrepo.tar', tmp_path_2) as path: | ||
testrepo = pygit2.Repository(path) | ||
return testrepo | ||
|
||
def confirm_same_repo_after_packing(testrepo, tmp_path, pack_delegate): | ||
# Helper method to confirm the contents of two repos before and after packing | ||
pack_repo = setup_second_repo(tmp_path) | ||
|
||
objects_dir = os.path.join(pack_repo.path, 'objects') | ||
rmtree(objects_dir) | ||
pack_path = os.path.join(pack_repo.path, 'objects', 'pack') | ||
os.makedirs(pack_path) | ||
|
||
# assert that the number of written objects is the same as the number of objects in the repo | ||
written_objects = testrepo.pack(pack_path, pack_delegate=pack_delegate) | ||
assert written_objects == len([obj for obj in testrepo]) | ||
|
||
|
||
# assert that the number of objects in the pack repo is the same as the original repo | ||
orig_objects = [obj for obj in testrepo.odb] | ||
packed_objects = [obj for obj in pack_repo.odb] | ||
assert len(packed_objects) == len(orig_objects) | ||
|
||
# assert that the objects in the packed repo are the same objects as the original repo | ||
for i, obj in enumerate(orig_objects): | ||
assert pack_repo[obj].type == testrepo[obj].type | ||
assert pack_repo[obj].read_raw() == testrepo[obj].read_raw() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You are not expecting a Git object here, but an Object id, so the argument
git_object
is misleading. I would call itoid
but then you had to rename the local variable with the same name, maybegit_oid
?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I've renamed it here and the other methods which call it