From ef9c792f54cd59b50a6e874c2e643a80bcaf8ee6 Mon Sep 17 00:00:00 2001 From: Daniel Widerin Date: Sat, 9 Jan 2016 12:41:30 +0100 Subject: [PATCH 01/17] Add GitLab repo sync and webhook support --- media/css/core.css | 6 + readthedocs/oauth/services/__init__.py | 6 +- readthedocs/oauth/services/gitlab.py | 222 +++++++++++++++++++++++++ readthedocs/settings/base.py | 1 + 4 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 readthedocs/oauth/services/gitlab.py diff --git a/media/css/core.css b/media/css/core.css index 6e2060514e9..a2aa0501ff0 100644 --- a/media/css/core.css +++ b/media/css/core.css @@ -672,6 +672,12 @@ a.socialaccount-provider.github:before { content: "\f09b"; } +div.project-import-remote form.import-connect-gitlab button:before, +a.socialaccount-provider.gitlab:before { + font-family: FontAwesome; + content: "\f1d3"; +} + div.project-import-remote form.import-connect-bitbucket button:before, a.socialaccount-provider.bitbucket:before, a.socialaccount-provider.bitbucket_oauth2:before { diff --git a/readthedocs/oauth/services/__init__.py b/readthedocs/oauth/services/__init__.py index 37ff9ca7851..5acfb6126b8 100644 --- a/readthedocs/oauth/services/__init__.py +++ b/readthedocs/oauth/services/__init__.py @@ -10,4 +10,8 @@ getattr(settings, 'OAUTH_BITBUCKET_SERVICE', 'readthedocs.oauth.services.bitbucket.BitbucketService')) -registry = [GitHubService, BitbucketService] +GitLabService = import_by_path( + getattr(settings, 'OAUTH_GITLAB_SERVICE', + 'readthedocs.oauth.services.gitlab.GitLabService')) + +registry = [GitHubService, BitbucketService, GitLabService] diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py new file mode 100644 index 00000000000..0e66b409ff2 --- /dev/null +++ b/readthedocs/oauth/services/gitlab.py @@ -0,0 +1,222 @@ +"""OAuth utility functions""" + +import logging +import json +import re + +from django.conf import settings +from requests.exceptions import RequestException +from allauth.socialaccount.models import SocialToken +from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter + +from readthedocs.restapi.client import api + +from ..models import RemoteOrganization, RemoteRepository +from .base import Service, DEFAULT_PRIVACY_LEVEL + + +log = logging.getLogger(__name__) + + +class GitLabService(Service): + """Provider service for GitLab""" + + adapter = GitLabOAuth2Adapter + url_pattern = re.compile(re.escape(adapter.provider_base_url)) + + def paginate(self, url, **kwargs): + """Combines return from GitLab pagination. GitLab uses + LinkHeaders, see: http://www.w3.org/wiki/LinkHeader + + :param url: start url to get the data from. + :param kwargs: optional parameters passed to .get() method + + See http://doc.gitlab.com/ce/api/README.html#pagination + """ + resp = self.get_session().get(url, data=kwargs) + result = resp.json() + next_url = resp.links.get('next', {}).get('url') + if next_url: + result.extend(self.paginate(next_url, **kwargs)) + return result + + def sync(self): + """Sync repositories from GitLab API""" + org = None + repos = self.paginate( + '{url}/api/v3/projects'.format(url=self.adapter.provider_base_url), + per_page=100, + order_by='path', + sort='asc' + ) + for repo in repos: + # Skip archived repositories + if repo.get('archived', False): + continue + if not org or org.slug != repo['namespace']['id']: + org = self.create_organization(repo['namespace']) + + self.create_repository(repo, organization=org) + + def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, + organization=None): + """Update or create a repository from GitLab API response + + :param fields: dictionary of response data from API + :param privacy: privacy level to support + :param organization: remote organization to associate with + :type organization: RemoteOrganization + :rtype: RemoteRepository + """ + # See: http://doc.gitlab.com/ce/api/projects.html#projects + repo_is_public = fields['visibility_level'] == 20 + + def is_owned_by(owner_id): + return self.account.extra_data['id'] == owner_id + + if privacy == 'private' or (repo_is_public and privacy == 'public'): + try: + repo = RemoteRepository.objects.get( + full_name=fields['name'], + users=self.user, + account=self.account, + ) + except RemoteRepository.DoesNotExist: + repo = RemoteRepository.objects.create( + full_name=fields['name'], + account=self.account, + ) + repo.users.add(self.user) + + if repo.organization and repo.organization != organization: + log.debug('Not importing %s because mismatched orgs' % + fields['name']) + return None + else: + repo.organization = organization + repo.name = fields['name'] + repo.full_name = fields['name_with_namespace'] + repo.description = fields['description'] + repo.ssh_url = fields['ssh_url_to_repo'] + repo.html_url = fields['web_url'] + repo.private = not fields['public'] + repo.clone_url = fields['http_url_to_repo'] + repo.admin = not repo_is_public + if not repo.admin and organization: + repo.admin = is_owned_by(fields['owner']['id']) + repo.vcs = 'git' + repo.account = self.account + repo.avatar_url = fields.get('avatar_url') + repo.json = json.dumps(fields) + repo.save() + return repo + else: + log.info( + 'Not importing {0} because mismatched type: public={1}'.format( + fields['name_with_namespace'], + fields['public'], + ) + ) + + def create_organization(self, fields): + """Update or create remote organization from GitLab API response + + :param fields: dictionary response of data from API + :rtype: RemoteOrganization + """ + try: + organization = RemoteOrganization.objects.get( + slug=fields.get('path'), + users=self.user, + account=self.account, + ) + except RemoteOrganization.DoesNotExist: + organization = RemoteOrganization.objects.create( + slug=fields.get('path'), + account=self.account, + ) + organization.users.add(self.user) + + organization.name = fields.get('name') + organization.account = self.account + organization.url = '{url}/{path}'.format( + url=self.adapter.provider_base_url, path=fields.get('path') + ) + if fields.get('avatar'): + organization.avatar_url = '{url}/{avatar}'.format( + url=self.adapter.provider_base_url, + avatar=fields['avatar']['url'], + ) + organization.json = json.dumps(fields) + organization.save() + return organization + + def setup_webhook(self, project): + """Set up GitLab project webhook for project + + :param project: project to set up webhook for + :type project: Project + :returns: boolean based on webhook set up success + :rtype: bool + """ + session = self.get_session() + + # See: http://doc.gitlab.com/ce/api/projects.html#add-project-hook + data = json.dumps({ + 'id': 'readthedocs', + 'push_events': True, + 'issues_events': False, + 'merge_requests_events': False, + 'note_events': False, + 'tag_push_events': True, + 'url': 'https://{0}/gitlab'.format(settings.PRODUCTION_DOMAIN), + }) + resp = None + try: + repositories = RemoteRepository.objects.filter( + clone_url=project.vcs_repo().repo_url + ) + assert repositories + repo_id = repositories[0].get_serialized()['id'] + resp = session.post( + '{url}/api/v3/projects/{repo_id}/hooks'.format( + url=self.adapter.provider_base_url, + repo_id=repo_id, + ), + data=data, + headers={'content-type': 'application/json'} + ) + if resp.status_code == 201: + log.info('GitLab webhook creation successful for project: %s', # noqa + project) + return True + except (AssertionError, RemoteRepository.DoesNotExist) as ex: + log.error('GitLab remote repository not found', exc_info=ex) + except RequestException as ex: + pass + else: + ex = False + + log.error('GitLab webhook creation failed for project: %s', # noqa + project, exc_info=ex) + + @classmethod + def get_token_for_project(cls, project, force_local=False): + """Get access token for project by iterating over project users""" + # TODO why does this only target GitHub? + if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False): + return None + token = None + try: + if getattr(settings, 'DONT_HIT_DB', True) and not force_local: + token = api.project(project.pk).token().get()['token'] + else: + for user in project.users.all(): + tokens = SocialToken.objects.filter( + account__user=user, + app__provider=cls.adapter.provider_id) + if tokens.exists(): + token = tokens[0].token + except Exception: + log.error('Failed to get token for user', exc_info=True) + return token diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 437af36602d..e64bd1daf37 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -108,6 +108,7 @@ def INSTALLED_APPS(self): # noqa 'allauth.account', 'allauth.socialaccount', 'allauth.socialaccount.providers.github', + 'allauth.socialaccount.providers.gitlab', 'allauth.socialaccount.providers.bitbucket', 'allauth.socialaccount.providers.bitbucket_oauth2', ] From cf5c47c45ab322f00c6cadc9efbbfa5a0ce97cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Thu, 16 Jun 2016 07:57:18 +0200 Subject: [PATCH 02/17] oauth: gitlab: use unicode whenever format is used --- readthedocs/oauth/services/gitlab.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 0e66b409ff2..8566546d800 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -44,7 +44,7 @@ def sync(self): """Sync repositories from GitLab API""" org = None repos = self.paginate( - '{url}/api/v3/projects'.format(url=self.adapter.provider_base_url), + u'{url}/api/v3/projects'.format(url=self.adapter.provider_base_url), per_page=100, order_by='path', sort='asc' @@ -112,7 +112,7 @@ def is_owned_by(owner_id): return repo else: log.info( - 'Not importing {0} because mismatched type: public={1}'.format( + u'Not importing {0} because mismatched type: public={1}'.format( fields['name_with_namespace'], fields['public'], ) @@ -139,11 +139,11 @@ def create_organization(self, fields): organization.name = fields.get('name') organization.account = self.account - organization.url = '{url}/{path}'.format( + organization.url = u'{url}/{path}'.format( url=self.adapter.provider_base_url, path=fields.get('path') ) if fields.get('avatar'): - organization.avatar_url = '{url}/{avatar}'.format( + organization.avatar_url = u'{url}/{avatar}'.format( url=self.adapter.provider_base_url, avatar=fields['avatar']['url'], ) @@ -169,7 +169,7 @@ def setup_webhook(self, project): 'merge_requests_events': False, 'note_events': False, 'tag_push_events': True, - 'url': 'https://{0}/gitlab'.format(settings.PRODUCTION_DOMAIN), + 'url': u'https://{0}/gitlab'.format(settings.PRODUCTION_DOMAIN), }) resp = None try: @@ -179,7 +179,7 @@ def setup_webhook(self, project): assert repositories repo_id = repositories[0].get_serialized()['id'] resp = session.post( - '{url}/api/v3/projects/{repo_id}/hooks'.format( + u'{url}/api/v3/projects/{repo_id}/hooks'.format( url=self.adapter.provider_base_url, repo_id=repo_id, ), From 2356d845c3dda3755618f91852419e4dccbf5c73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Mon, 20 Jun 2016 06:47:48 +0200 Subject: [PATCH 03/17] oauth: gitlab: prevent duplicate entries after sync --- readthedocs/oauth/services/gitlab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 8566546d800..5c21b737090 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -77,13 +77,13 @@ def is_owned_by(owner_id): if privacy == 'private' or (repo_is_public and privacy == 'public'): try: repo = RemoteRepository.objects.get( - full_name=fields['name'], + full_name=fields['name_with_namespace'], users=self.user, account=self.account, ) except RemoteRepository.DoesNotExist: repo = RemoteRepository.objects.create( - full_name=fields['name'], + full_name=fields['name_with_namespace'], account=self.account, ) repo.users.add(self.user) From 7113e94b568ef28aaff9e7b1c7e18304142c76e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Mon, 20 Jun 2016 06:50:29 +0200 Subject: [PATCH 04/17] oauth: gitlab: avoid KeyError when owner is missing --- readthedocs/oauth/services/gitlab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 5c21b737090..3475021971d 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -102,7 +102,7 @@ def is_owned_by(owner_id): repo.private = not fields['public'] repo.clone_url = fields['http_url_to_repo'] repo.admin = not repo_is_public - if not repo.admin and organization: + if not repo.admin and 'owner' in fields: repo.admin = is_owned_by(fields['owner']['id']) repo.vcs = 'git' repo.account = self.account From 5d7d6c1649864b9eff9580a7336f81a8288a4164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Mon, 20 Jun 2016 06:53:36 +0200 Subject: [PATCH 05/17] oauth: gitlab: setup_webhook must return a tuple --- readthedocs/oauth/services/gitlab.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 3475021971d..33dcc1efc4b 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -189,7 +189,7 @@ def setup_webhook(self, project): if resp.status_code == 201: log.info('GitLab webhook creation successful for project: %s', # noqa project) - return True + return (True, resp) except (AssertionError, RemoteRepository.DoesNotExist) as ex: log.error('GitLab remote repository not found', exc_info=ex) except RequestException as ex: @@ -199,6 +199,7 @@ def setup_webhook(self, project): log.error('GitLab webhook creation failed for project: %s', # noqa project, exc_info=ex) + return (False, resp) @classmethod def get_token_for_project(cls, project, force_local=False): From f9df35e3cba0a71961057d18091ce70ed4fd98a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Mon, 20 Jun 2016 07:08:23 +0200 Subject: [PATCH 06/17] oauth: gitlab: set a default avatar if none is returned --- media/images/fa-bookmark.svg | 1 + media/images/fa-users.svg | 1 + readthedocs/oauth/services/gitlab.py | 17 ++++++++++++++--- 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 media/images/fa-bookmark.svg create mode 100644 media/images/fa-users.svg diff --git a/media/images/fa-bookmark.svg b/media/images/fa-bookmark.svg new file mode 100644 index 00000000000..0daca66b42b --- /dev/null +++ b/media/images/fa-bookmark.svg @@ -0,0 +1 @@ + diff --git a/media/images/fa-users.svg b/media/images/fa-users.svg new file mode 100644 index 00000000000..7da2bd570b4 --- /dev/null +++ b/media/images/fa-users.svg @@ -0,0 +1 @@ + diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 33dcc1efc4b..e772c17fa5e 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -8,6 +8,7 @@ from requests.exceptions import RequestException from allauth.socialaccount.models import SocialToken from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter +from urlparse import urljoin from readthedocs.restapi.client import api @@ -23,6 +24,10 @@ class GitLabService(Service): adapter = GitLabOAuth2Adapter url_pattern = re.compile(re.escape(adapter.provider_base_url)) + default_avatar = { + 'repo': urljoin(settings.MEDIA_URL, 'images/fa-bookmark.svg'), + 'org': urljoin(settings.MEDIA_URL, 'images/fa-users.svg'), + } def paginate(self, url, **kwargs): """Combines return from GitLab pagination. GitLab uses @@ -106,7 +111,10 @@ def is_owned_by(owner_id): repo.admin = is_owned_by(fields['owner']['id']) repo.vcs = 'git' repo.account = self.account - repo.avatar_url = fields.get('avatar_url') + owner = fields.get('owner') or {} + repo.avatar_url = (fields.get('avatar_url') or + owner.get('avatar_url') or + self.default_avatar['repo']) repo.json = json.dumps(fields) repo.save() return repo @@ -142,11 +150,14 @@ def create_organization(self, fields): organization.url = u'{url}/{path}'.format( url=self.adapter.provider_base_url, path=fields.get('path') ) - if fields.get('avatar'): + avatar = fields.get('avatar') or {} + if avatar.get('url'): organization.avatar_url = u'{url}/{avatar}'.format( url=self.adapter.provider_base_url, - avatar=fields['avatar']['url'], + avatar=avatar.get('url'), ) + else: + organization.avatar_url = self.default_avatar['org'] organization.json = json.dumps(fields) organization.save() return organization From e4adb98e6c18a9de1636ea3e5bdbe05444d25eac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Mon, 20 Jun 2016 08:32:46 +0200 Subject: [PATCH 07/17] oauth: gitlab: use SSH url if repo is private --- readthedocs/oauth/services/gitlab.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index e772c17fa5e..0fe4b997342 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -105,8 +105,9 @@ def is_owned_by(owner_id): repo.ssh_url = fields['ssh_url_to_repo'] repo.html_url = fields['web_url'] repo.private = not fields['public'] - repo.clone_url = fields['http_url_to_repo'] repo.admin = not repo_is_public + repo.clone_url = (repo.admin and repo.ssh_url or + fields['http_url_to_repo']) if not repo.admin and 'owner' in fields: repo.admin = is_owned_by(fields['owner']['id']) repo.vcs = 'git' From da7471f553fe821c2c02761bf82ecad9f9341102 Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Fri, 5 Aug 2016 11:33:32 +0200 Subject: [PATCH 08/17] Added tests for the GitLabService --- readthedocs/rtd_tests/tests/test_oauth.py | 135 +++++++++++++++++++++- 1 file changed, 133 insertions(+), 2 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index 9b879ca760f..16433942d64 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -1,12 +1,13 @@ from django.test import TestCase from django.contrib.auth.models import User -from allauth.socialaccount.models import SocialToken +from mock import Mock +from readthedocs.projects import constants from readthedocs.projects.models import Project -from readthedocs.oauth.services import GitHubService, BitbucketService from readthedocs.oauth.models import RemoteRepository, RemoteOrganization +from readthedocs.oauth.services import GitHubService, BitbucketService, GitLabService class GitHubOAuthTests(TestCase): @@ -279,3 +280,133 @@ def test_import_with_no_token(self): '''User without a Bitbucket SocialToken does not return a service''' services = BitbucketService.for_user(self.user) self.assertEqual(services, []) + + +class GitLabOAuthTests(TestCase): + + fixtures = ["eric", "test_data"] + + repo_response_data = { + 'forks_count': 12, + 'container_registry_enabled': None, + 'web_url': 'https://gitlab.com/testorga/testrepo', + 'wiki_enabled': True, + 'public_builds': True, + 'id': 2, + 'merge_requests_enabled': True, + 'archived': False, + 'snippets_enabled': False, + 'http_url_to_repo': 'https://gitlab.com/testorga/testrepo.git', + 'namespace': { + 'share_with_group_lock': False, + 'name': 'Test Orga', + 'created_at': '2014-07-11T13:38:53.510Z', + 'description': '', + 'updated_at': '2014-07-11T13:38:53.510Z', + 'avatar': { + 'url': None + }, + 'path': 'testorga', + 'visibility_level': 20, + 'id': 5, + 'owner_id': None + }, + 'star_count': 0, + 'avatar_url': 'http://placekitten.com/50/50', + 'issues_enabled': True, + 'path_with_namespace': 'testorga/testrepo', + 'public': True, + 'description': 'Test Repo', + 'default_branch': 'master', + 'ssh_url_to_repo': 'git@gitlab.com:testorga/testrepo.git', + 'path': 'testrepo', + 'visibility_level': 20, + 'permissions': { + 'group_access': { + 'notification_level': 3, + 'access_level': 40 + }, + 'project_access': None + }, + 'open_issues_count': 2, + 'last_activity_at': '2016-03-01T09:22:34.344Z', + 'name': 'testrepo', + 'name_with_namespace': 'testorga / testrepo', + 'created_at': '2015-11-02T13:52:42.821Z', + 'builds_enabled': True, + 'creator_id': 5, + 'shared_runners_enabled': True, + 'tag_list': [] + } + + def setUp(self): + self.client.login(username='eric', password='test') + self.user = User.objects.get(pk=1) + self.project = Project.objects.get(slug='pip') + self.org = RemoteOrganization.objects.create(slug='testorga', json='') + self.privacy = self.project.version_privacy_level + self.service = GitLabService(user=self.user, account=None) + + def get_private_repo_data(self): + """Manipulate repo response data to get private repo data.""" + data = self.repo_response_data.copy() + data.update({ + 'visibility_level': 10, + 'public': False, + }) + return data + + def test_make_project_pass(self): + repo = self.service.create_repository( + self.repo_response_data, organization=self.org, privacy=self.privacy) + self.assertIsInstance(repo, RemoteRepository) + self.assertEqual(repo.name, 'testrepo') + self.assertEqual(repo.full_name, 'testorga / testrepo') + self.assertEqual(repo.description, 'Test Repo') + self.assertEqual(repo.avatar_url, 'http://placekitten.com/50/50') + self.assertIn(self.user, repo.users.all()) + self.assertEqual(repo.organization, self.org) + self.assertEqual(repo.clone_url, 'https://gitlab.com/testorga/testrepo.git') + self.assertEqual(repo.ssh_url, 'git@gitlab.com:testorga/testrepo.git') + self.assertEqual(repo.html_url, 'https://gitlab.com/testorga/testrepo') + + def test_make_private_project_fail(self): + repo = self.service.create_repository( + self.get_private_repo_data(), organization=self.org, privacy=self.privacy) + self.assertIsNone(repo) + + def test_make_private_project_success(self): + repo = self.service.create_repository( + self.get_private_repo_data(), organization=self.org, privacy=constants.PRIVATE) + self.assertIsInstance(repo, RemoteRepository) + self.assertTrue(repo.private, True) + + def test_make_organization(self): + org = self.service.create_organization(self.repo_response_data['namespace']) + self.assertIsInstance(org, RemoteOrganization) + self.assertEqual(org.slug, 'testorga') + self.assertEqual(org.name, 'Test Orga') + self.assertEqual(org.avatar_url, '/media/images/fa-users.svg') + self.assertEqual(org.url, 'https://gitlab.com/testorga') + + def test_sync_skip_archived_repo(self): + data = self.repo_response_data + data['archived'] = True + create_repo_mock = Mock() + create_orga_mock = Mock() + setattr(self.service, 'paginate', Mock(return_value=[data])) + setattr(self.service, 'create_repository', create_repo_mock) + setattr(self.service, 'create_organization', create_orga_mock) + self.service.sync() + self.assertFalse(create_repo_mock.called) + self.assertFalse(create_orga_mock.called) + + def test_sync_create_repo_and_orga(self): + create_repo_mock = Mock() + create_orga_mock = Mock(return_value=self.org) + setattr(self.service, 'paginate', Mock(return_value=[self.repo_response_data])) + setattr(self.service, 'create_repository', create_repo_mock) + setattr(self.service, 'create_organization', create_orga_mock) + self.service.sync() + create_repo_mock.assert_called_once_with(self.repo_response_data, organization=self.org) + create_orga_mock.assert_called_once_with(self.repo_response_data['namespace']) From aec69413b1d57bdb9773c7b9c434f78c5fbcf7ff Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Fri, 5 Aug 2016 11:48:40 +0200 Subject: [PATCH 09/17] Added some documentation for the Gitlab integration --- docs/features.rst | 2 +- docs/webhooks.rst | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/features.rst b/docs/features.rst index b4dae131a58..6f1de62f0f4 100644 --- a/docs/features.rst +++ b/docs/features.rst @@ -13,7 +13,7 @@ More information can be found in the :doc:`vcs` page. Auto-updating ------------- -The :doc:`webhooks` page talks about the different ways you can ping RTD to let us know your project has been updated. We have official support for GitHub, and anywhere else we have a generic post-commit hook that allows you to POST to a URL to get your documentation built. +The :doc:`webhooks` page talks about the different ways you can ping RTD to let us know your project has been updated. We have official support for GitHub, Bitbucket and GitLab, and anywhere else we have a generic post-commit hook that allows you to POST to a URL to get your documentation built. Internationalization -------------------- diff --git a/docs/webhooks.rst b/docs/webhooks.rst index e912e15f529..926cc7512c3 100644 --- a/docs/webhooks.rst +++ b/docs/webhooks.rst @@ -9,7 +9,7 @@ worked with push knows, pushing a doc update to your repo and watching it get updated within seconds is an awesome feeling. GitHub ---------- +------ If your project is hosted on GitHub, you can easily add a hook that will rebuild your docs whenever you push updates: @@ -27,7 +27,7 @@ If you ever need to manually set the webhook on GitHub, you can point it at ``https://readthedocs.org/github``. Bitbucket ------------ +--------- If your project is hosted on Bitbucket, you can easily add a hook that will rebuild your docs whenever you push updates: @@ -40,6 +40,17 @@ your docs whenever you push updates: If you ever need to manually set the webhook on Bitbucket, you can point it at ``https://readthedocs.org/bitbucket``. +GitLab +------ + +If your project is hosted on GitLab, you can manually set the webhook on Gitlab and +point it at ``https://readthedocs.org/gitlab``: + +* Click the settings icon for your project +* Select "Webhooks" +* Enter the above URL, select "Push events" and "Enable SSL verification" +* Click "Add Webhook" + Others ------ From 299a2b54258ab589c1c08ec266f2cc93b2e5fd51 Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Fri, 5 Aug 2016 14:11:42 +0200 Subject: [PATCH 10/17] Improved url_pattern of the GitLabService to support private repositories --- readthedocs/oauth/services/gitlab.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 0fe4b997342..f29f7ec1de5 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -4,11 +4,15 @@ import json import re +try: + from urlparse import urljoin, urlparse +except ImportError: + from urllib.parse import urljoin, urlparse + from django.conf import settings from requests.exceptions import RequestException from allauth.socialaccount.models import SocialToken from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter -from urlparse import urljoin from readthedocs.restapi.client import api @@ -23,7 +27,9 @@ class GitLabService(Service): """Provider service for GitLab""" adapter = GitLabOAuth2Adapter - url_pattern = re.compile(re.escape(adapter.provider_base_url)) + # Just use the network location to determine if it's a GitLab project + # because private repos have another base url, eg. git@gitlab.example.com + url_pattern = re.compile(re.escape(urlparse(adapter.provider_base_url).netloc)) default_avatar = { 'repo': urljoin(settings.MEDIA_URL, 'images/fa-bookmark.svg'), 'org': urljoin(settings.MEDIA_URL, 'images/fa-users.svg'), From 8a1084844ce28503d9c028b3c7b0f39cf897e16f Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Fri, 5 Aug 2016 14:35:39 +0200 Subject: [PATCH 11/17] Fixed linting error --- readthedocs/oauth/services/gitlab.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index f29f7ec1de5..44c66d7b549 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -4,10 +4,7 @@ import json import re -try: - from urlparse import urljoin, urlparse -except ImportError: - from urllib.parse import urljoin, urlparse +from urlparse import urljoin, urlparse from django.conf import settings from requests.exceptions import RequestException From 97a4fb15f0144e04aaa797fe2c30382629ccf70f Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Wed, 10 Aug 2016 11:30:28 +0200 Subject: [PATCH 12/17] Simplified exception handling --- readthedocs/oauth/services/gitlab.py | 30 ++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 44c66d7b549..69edf5c5313 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -187,12 +187,14 @@ def setup_webhook(self, project): 'url': u'https://{0}/gitlab'.format(settings.PRODUCTION_DOMAIN), }) resp = None + repositories = RemoteRepository.objects.filter(clone_url=project.vcs_repo().repo_url) + + if not repositories.exists(): + log.error('GitLab remote repository not found') + return False, resp + + repo_id = repositories[0].get_serialized()['id'] try: - repositories = RemoteRepository.objects.filter( - clone_url=project.vcs_repo().repo_url - ) - assert repositories - repo_id = repositories[0].get_serialized()['id'] resp = session.post( u'{url}/api/v3/projects/{repo_id}/hooks'.format( url=self.adapter.provider_base_url, @@ -202,19 +204,13 @@ def setup_webhook(self, project): headers={'content-type': 'application/json'} ) if resp.status_code == 201: - log.info('GitLab webhook creation successful for project: %s', # noqa - project) - return (True, resp) - except (AssertionError, RemoteRepository.DoesNotExist) as ex: - log.error('GitLab remote repository not found', exc_info=ex) - except RequestException as ex: - pass + log.info('GitLab webhook creation successful for project: %s', project) + return True, resp + except RequestException: + log.error('GitLab webhook creation failed for project: %s', project, exc_info=True) else: - ex = False - - log.error('GitLab webhook creation failed for project: %s', # noqa - project, exc_info=ex) - return (False, resp) + log.error('GitLab webhook creation failed for project: %s', project) + return False, resp @classmethod def get_token_for_project(cls, project, force_local=False): From dff4d7db45630b1899bc1c2a0d1b8a01cc914c95 Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Wed, 10 Aug 2016 11:31:25 +0200 Subject: [PATCH 13/17] Use PRODUCTION_DOMAIN as webhook id --- readthedocs/oauth/services/gitlab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 69edf5c5313..f7418df57b7 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -178,7 +178,7 @@ def setup_webhook(self, project): # See: http://doc.gitlab.com/ce/api/projects.html#add-project-hook data = json.dumps({ - 'id': 'readthedocs', + 'id': settings.PRODUCTION_DOMAIN, 'push_events': True, 'issues_events': False, 'merge_requests_events': False, From 4fbe6bbb142dc814f463829d7525bbd2a87acd35 Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Wed, 10 Aug 2016 11:36:09 +0200 Subject: [PATCH 14/17] Make imports python3 compatible --- readthedocs/oauth/services/gitlab.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index f7418df57b7..6e668f7c036 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -4,7 +4,10 @@ import json import re -from urlparse import urljoin, urlparse +try: + from urlparse import urljoin, urlparse +except ImportError: + from urllib.parse import urljoin, urlparse # noqa from django.conf import settings from requests.exceptions import RequestException From 50285fe2dd4dc1a77620ccd2b9b62b98f3619054 Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Wed, 10 Aug 2016 12:01:35 +0200 Subject: [PATCH 15/17] Use the GitLab project ID, not the PRODUCTION_DOMAIN --- readthedocs/oauth/services/gitlab.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 6e668f7c036..bed20faf3fd 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -178,10 +178,17 @@ def setup_webhook(self, project): :rtype: bool """ session = self.get_session() + resp = None + repositories = RemoteRepository.objects.filter(clone_url=project.vcs_repo().repo_url) + if not repositories.exists(): + log.error('GitLab remote repository not found') + return False, resp + + repo_id = repositories[0].get_serialized()['id'] # See: http://doc.gitlab.com/ce/api/projects.html#add-project-hook data = json.dumps({ - 'id': settings.PRODUCTION_DOMAIN, + 'id': repo_id, 'push_events': True, 'issues_events': False, 'merge_requests_events': False, @@ -189,14 +196,7 @@ def setup_webhook(self, project): 'tag_push_events': True, 'url': u'https://{0}/gitlab'.format(settings.PRODUCTION_DOMAIN), }) - resp = None - repositories = RemoteRepository.objects.filter(clone_url=project.vcs_repo().repo_url) - if not repositories.exists(): - log.error('GitLab remote repository not found') - return False, resp - - repo_id = repositories[0].get_serialized()['id'] try: resp = session.post( u'{url}/api/v3/projects/{repo_id}/hooks'.format( From 6c8d1c30c0c9dd326dd46c0de39a77358b1d5510 Mon Sep 17 00:00:00 2001 From: Daniel Widerin Date: Wed, 19 Oct 2016 18:12:32 +0200 Subject: [PATCH 16/17] Update urls as mentioned in comments --- readthedocs/oauth/services/gitlab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index bed20faf3fd..f1b94d3a587 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -42,7 +42,7 @@ def paginate(self, url, **kwargs): :param url: start url to get the data from. :param kwargs: optional parameters passed to .get() method - See http://doc.gitlab.com/ce/api/README.html#pagination + See https://docs.gitlab.com/ce/api/README.html#pagination """ resp = self.get_session().get(url, data=kwargs) result = resp.json() @@ -79,7 +79,7 @@ def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, :type organization: RemoteOrganization :rtype: RemoteRepository """ - # See: http://doc.gitlab.com/ce/api/projects.html#projects + # See: https://docs.gitlab.com/ce/api/projects.html#projects repo_is_public = fields['visibility_level'] == 20 def is_owned_by(owner_id): From 0cb1ed5fab920fb5ddac25ca6e256a1537f3f36e Mon Sep 17 00:00:00 2001 From: Daniel Widerin Date: Wed, 19 Oct 2016 18:17:47 +0200 Subject: [PATCH 17/17] Update webhook docs as mentioned in comments --- docs/webhooks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/webhooks.rst b/docs/webhooks.rst index 926cc7512c3..1220f8b641d 100644 --- a/docs/webhooks.rst +++ b/docs/webhooks.rst @@ -43,8 +43,8 @@ you can point it at ``https://readthedocs.org/bitbucket``. GitLab ------ -If your project is hosted on GitLab, you can manually set the webhook on Gitlab and -point it at ``https://readthedocs.org/gitlab``: +If your project is hosted on GitLab.com, you can manually set the webhook on +Gitlab.com and point it at ``https://readthedocs.org/gitlab``: * Click the settings icon for your project * Select "Webhooks"