Bounty Payout Milestone 1-Review

app/app/management/commands/sync_orgs_repos.py

@@ -0,0 +1,179 @@
+from django.contrib.auth.models import Group, User
+from django.core.management.base import BaseCommand
+
+from app.utils import sync_profile
+from dashboard.models import Organization, Profile, Repo
+from git.utils import get_organization, get_repo, get_user
+
+
+class Command(BaseCommand):
+    help = 'Synchronizes Organizations and Repo roles to members'
+
+    def handle(self, *args, **options):
+
+        try:
+            print("Loading Users....")
+            all_users = Profile.objects.filter(
+                user__is_active=True
+            ).prefetch_related('user')
+
+            synced = []
+            orgs_synced = []
+
+            def recursive_sync(lsynced, handle):
+                try:
+
+                    if handle not in lsynced:
+
+                        print(f'Syncing User Handle: {handle}')
+                        profile = sync_profile(handle)
+                        access_token = profile.user.social_auth.filter(provider='github').latest('pk').access_token
+                        print('Removing Stale Organizations and Groups')
+                        remove_org_groups = [x for x in profile.profile_organizations.all() if
+                                             x.name not in profile.organizations]
+                        for y in remove_org_groups:
+                            profile.profile_organizations.remove(y)
+                            profile.user.groups.filter(name__contains=y.name).delete()
+                            print(f'Removing: {profile.handle} from Organization: {y.name} ')
+
+                        user_access_repos = get_repo(handle, '/repos', (handle, access_token), is_user=True)
+                        # Question around user repo acccess if we can't get user repos, should we assume all repos are no longer available in the platform?
+
+                        if 'message' in user_access_repos:
+                            print(user_access_repos['message'])
+                            return lsynced
+
+                        current_user_repos = []
+                        for y in user_access_repos:
+                            current_user_repos.append(y['name'])
+
+                        remove_user_repos_names = [x for x in profile.repos.all() if
+                                                   x.name not in current_user_repos]
+
+                        remove_user_repos = Repo.objects.filter(name__in=remove_user_repos_names,
+                                                                profile__handle=handle)
+                        for y in remove_user_repos:
+                            profile.repos.remove(y)
+
+                        lsynced.append(handle)
+                    else:
+                        return lsynced
+
+                    members_to_sync = []
+                    if profile.organizations is None:
+                        print("no organizations to sync")
+                        return []
+                    for org in profile.organizations:
+                        if org in orgs_synced:
+                            continue
+
+                        orgs_synced.append(org)
+                        db_org = Organization.objects.get_or_create(name=org)[0]
+
+                        print(f'Syncing Organization: {db_org.name}')
+                        profile.profile_organizations.add(db_org)
+                        org_members = get_organization(
+                            db_org.name,
+                            '/members'
+                        )
+
+                        if 'message' in org_members:
+                            print(org_members['message'])
+                            continue
+
+                        for member in org_members:
+
+                            try:
+
+                                member_profile_obj = Profile.objects.get(
+                                    handle=member['login'],
+                                    user__is_active=True
+                                )
+                                membership = get_organization(
+                                    db_org.name,
+                                    f'/memberships/{member["login"]}'
+                                )
+                                role = membership['role'] if not None else "member"
+                                db_group = Group.objects.get_or_create(name=f'{db_org.name}-role-{role}')[0]
+                                db_org.groups.add(db_group)
+                                member_profile_obj.user.groups.add(db_group)
+                                members_to_sync.append(member['login'])
+                                lsynced = recursive_sync(lsynced, member['login'])
+                            except Exception as e:
+                                print(f'Profile for Github Handle does not exist: {member["login"]}')
+                                continue
+
+                        org_repos = get_organization(
+                            db_org.name,
+                            '/repos'
+                        )
+
+                        if 'message' in org_repos:
+                            print(org_repos['message'])
+                            continue
+
+                        for repo in org_repos:
+                            db_repo = Repo.objects.get_or_create(name=repo['name'])[0]
+                            db_org.repos.add(db_repo)
+                            print(db_org)
+                            print(f'Syncing Repo: {db_repo.name}')
+                            repo_collabs = get_repo(
+                                repo['full_name'],
+                                '/collaborators',
+                                (handle, access_token)
+                            )
+                            if 'message' in repo_collabs:
+                                print(repo_collabs['message'])
+                                continue
+
+                            for collaborator in repo_collabs:
+                                try:
+                                    member_user_profile = Profile.objects.get(handle=collaborator['login'],
+                                                                              user__is_active=True)
+
+                                    if collaborator['permissions']['admin']:
+                                        permission = "admin"
+                                    elif collaborator['permissions']['push']:
+                                        permission = "write"
+                                    elif collaborator['permissions']['pull']:
+                                        permission = "pull"
+                                    else:
+                                        permission = "none"
+
+                                    db_group = Group.objects.get_or_create(
+                                        name=f'{db_org.name}-repo-{repo["name"]}-{permission}')[0]
+                                    db_org.groups.add(db_group)
+                                    member_user_profile.user.groups.add(db_group)
+                                    member_user_profile.repos.add(db_repo)
+
+                                    if collaborator['login'] not in members_to_sync or collaborator[
+                                        'login'] not in lsynced:
+                                        members_to_sync.append(collaborator['login'])
+                                    for x in members_to_sync:
+                                        print(x)
+                                        try:
+                                            lsynced = lsynced + recursive_sync(lsynced, x)
+                                        except ValueError as members_loop_exec:
+
+                                            print(members_loop_exec)
+                                except Exception as e:
+                                    print(e)
+                                    continue
+
+                        return lsynced
+                except Exception as exc:
+                    print(f'Unhandled Exception occurred: {exc}')
+
+            for profile in all_users:
+                try:
+                    if profile.handle is not None:
+                        synced = recursive_sync(synced, profile.handle)
+                except ValueError as loop_exc:
+                    print(f'Error syncing user id:{profile.user.id}')
+                    print(loop_exc)
+                except Exception as e:
+                    print(e)
+
+            print("Sync Completed")
+        except ValueError as e:
+            print(e)

app/app/settings.py

@@ -529,7 +529,7 @@
 SOCIAL_AUTH_GITHUB_SECRET = GITHUB_CLIENT_SECRET
 SOCIAL_AUTH_POSTGRES_JSONFIELD = True
 SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = ['username', 'first_name', 'last_name', 'email']
-SOCIAL_AUTH_GITHUB_SCOPE = ['read:public_repo', 'read:user', 'user:email', ]
+SOCIAL_AUTH_GITHUB_SCOPE = ['read:public_repo', 'read:user', 'user:email', 'read:org', 'repo']
 SOCIAL_AUTH_SANITIZE_REDIRECTS = True
 
 SOCIAL_AUTH_PIPELINE = (

app/app/utils.py

@@ -183,7 +183,7 @@ def get_upload_filename(instance, filename):
 
 
 def sync_profile(handle, user=None, hide_profile=True):
-    from dashboard.models import Profile
+    from dashboard.models import Profile, Organization
     handle = handle.strip().replace('@', '').lower()
     data = get_user(handle)
     email = ''
@@ -207,9 +207,9 @@ def sync_profile(handle, user=None, hide_profile=True):
     # store the org info in postgres
     try:
         profile, created = Profile.objects.update_or_create(handle=handle, defaults=defaults)
-        print("Profile:", profile, "- created" if created else "- updated")
-        orgs = get_user(handle, '/orgs')
+        orgs = get_user(handle, '', scope='orgs')
         profile.organizations = [ele['login'] for ele in orgs]
+        print("Profile:", profile, "- created" if created else "- updated")
         keywords = []
         for repo in profile.repos_data_lite:
             language = repo.get('language') if repo.get('language') else ''

app/dashboard/migrations/0054_auto_20190930_1525.py

@@ -0,0 +1,51 @@
+# Generated by Django 2.2.4 on 2019-09-30 15:25
+
+from django.db import migrations, models
+import economy.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('auth', '0011_update_proxy_permissions'),
+        ('dashboard', '0053_auto_20190920_1816'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Repo',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_on', models.DateTimeField(db_index=True, default=economy.models.get_time)),
+                ('modified_on', models.DateTimeField(default=economy.models.get_time)),
+                ('name', models.CharField(max_length=255)),
+            ],
+            options={
+                'ordering': ('name',),
+            },
+        ),
+        migrations.CreateModel(
+            name='Organization',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_on', models.DateTimeField(db_index=True, default=economy.models.get_time)),
+                ('modified_on', models.DateTimeField(default=economy.models.get_time)),
+                ('name', models.CharField(max_length=255)),
+                ('groups', models.ManyToManyField(blank=True, to='auth.Group')),
+                ('repos', models.ManyToManyField(blank=True, to='dashboard.Repo')),
+            ],
+            options={
+                'ordering': ('name',),
+            },
+        ),
+        migrations.AddField(
+            model_name='profile',
+            name='profile_organizations',
+            field=models.ManyToManyField(blank=True, to='dashboard.Organization'),
+        ),
+        migrations.AddField(
+            model_name='profile',
+            name='repos',
+            field=models.ManyToManyField(blank=True, to='dashboard.Repo'),
+        ),
+    ]

app/dashboard/models.py

@@ -2140,6 +2140,28 @@ def hidden(self):
         return self.filter(hide_profile=True)
 
 
+class Repo(SuperModel):
+    name = models.CharField(max_length=255)
+
+    class Meta:
+        ordering = ('name',)
+
+    def __str__(self):
+        return self.name
+
+
+class Organization(SuperModel):
+    name = models.CharField(max_length=255)
+    groups = models.ManyToManyField('auth.Group', blank=True)
+    repos = models.ManyToManyField(Repo, blank=True)
+
+    class Meta:
+        ordering = ('name',)
+
+    def __str__(self):
+        return self.name
+
+
 class Profile(SuperModel):
     """Define the structure of the user profile.
 
@@ -2188,6 +2210,8 @@ class Profile(SuperModel):
     )
     keywords = ArrayField(models.CharField(max_length=200), blank=True, default=list)
     organizations = ArrayField(models.CharField(max_length=200), blank=True, default=list)
+    profile_organizations = models.ManyToManyField(Organization, blank=True)
+    repos = models.ManyToManyField(Repo, blank=True)
     form_submission_records = JSONField(default=list, blank=True)
     max_num_issues_start_work = models.IntegerField(default=3)
     preferred_payout_address = models.CharField(max_length=255, default='', blank=True)

app/git/utils.py

@@ -134,7 +134,7 @@ def check_github(profile):
 
 
 def search_github(q):
-    params = (('q', q), ('sort', 'updated'), )
+    params = (('q', q), ('sort', 'updated'),)
     response = requests.get('https://api.github.com/search/users', headers=HEADERS, params=params)
     return response.json()
 
@@ -229,7 +229,7 @@ def get_auth_url(redirect_uri='/'):
         str: The Github authentication URL.
 
     """
-    github_callback = reverse('social:begin', args=('github', ))
+    github_callback = reverse('social:begin', args=('github',))
     redirect_params = {'next': BASE_URI + redirect_uri}
     redirect_uri = urlencode(redirect_params, quote_via=quote_plus)
 
@@ -326,7 +326,7 @@ def get_github_event_emails(oauth_token, username):
                 name = author.get('name', {})
                 if name and username and user_name:
                     append_email = name.lower() == username.lower() or name.lower() == user_name.lower() \
-                        and email and 'noreply.github.com' not in email
+                                   and email and 'noreply.github.com' not in email
                     if append_email:
                         emails.append(email)
 
@@ -395,7 +395,7 @@ def search(query):
         request.Response: The github search response.
 
     """
-    params = (('q', query), ('sort', 'updated'), )
+    params = (('q', query), ('sort', 'updated'),)
 
     try:
         response = requests.get('https://api.github.com/search/users', auth=_AUTH, headers=V3HEADERS, params=params)
@@ -467,7 +467,8 @@ def get_issue_comments(owner, repo, issue=None, comment_id=None):
     params = {
         'sort': 'created',
         'direction': 'desc',
-        'per_page': 100,  # TODO traverse/concat pages: https://developer.github.com/v3/guides/traversing-with-pagination/
+        'per_page': 100,
+    # TODO traverse/concat pages: https://developer.github.com/v3/guides/traversing-with-pagination/
     }
     if issue:
         if comment_id:
@@ -590,11 +591,45 @@ def get_interested_actions(github_url, username, email=''):
     return actions_by_interested_party
 
 
-def get_user(user, sub_path=''):
+def get_user(user, sub_path='', scope='', auth=_AUTH):
     """Get the github user details."""
-    user = user.replace('@', '')
-    url = f'https://api.github.com/users/{user}{sub_path}'
-    response = requests.get(url, auth=_AUTH, headers=HEADERS)
+    if scope is not '':
+        print(scope)
+        url = f'https://api.github.com/user/{scope}'
+    else:
+        user = user.replace('@', '')
+        url = f'https://api.github.com/users/{user}{sub_path}'
+
+    response = requests.get(url, auth=auth, headers=HEADERS)
+
+    try:
+        response_dict = response.json()
+    except JSONDecodeError:
+        response_dict = {}
+    return response_dict
+
+
+def get_organization(org, sub_path='', auth=_AUTH):
+    """Get the github user details."""
+    org = org.replace('@', '')
+    url = f'https://api.github.com/orgs/{org}{sub_path}'
+    response = requests.get(url, auth=auth, headers=HEADERS)
+    try:
+        response_dict = response.json()
+    except JSONDecodeError:
+        response_dict = {}
+    return response_dict
+
+
+def get_repo(repo_full_name, sub_path='', auth=_AUTH, is_user=False):
+    """Get the github user details."""
+    repo_full_name = repo_full_name.replace('@', '')
+    if is_user:
+        url = f'https://api.github.com/user/repos'
+    else:
+        url = f'https://api.github.com/repos/{repo_full_name}{sub_path}'
+
+    response = requests.get(url, auth=auth, headers=HEADERS)
 
     try:
         response_dict = response.json()