Settings org 2

app/app/management/commands/sync_orgs_repos.py

@@ -0,0 +1,187 @@
+from django.contrib.auth.models import Group, User
+from django.core.management.base import BaseCommand
+
+from app.utils import sync_profile
+from dashboard.models import Organization, Profile, Repo
+from git.utils import get_organization, get_repo, get_user
+
+
+class Command(BaseCommand):
+    help = 'Synchronizes Organizations and Repo roles to members'
+
+    def handle(self, *args, **options):
+
+        try:
+            print("Loading Users....")
+            all_users = Profile.objects.filter(
+                user__is_active=True
+            ).prefetch_related('user')
+
+            synced = []
+            orgs_synced = []
+
+            def recursive_sync(lsynced, handle):
+                try:
+
+                    if handle not in lsynced:
+
+                        print(f'Syncing User Handle: {handle}')
+                        profile = sync_profile(handle)
+                        access_token = profile.user.social_auth.filter(provider='github').latest('pk').access_token
+                        print('Removing Stale Organizations and Groups')
+                        remove_org_groups = [x for x in profile.profile_organizations.all() if
+                                             x.name not in profile.organizations]
+                        for y in remove_org_groups:
+                            profile.profile_organizations.remove(y)
+                            profile.user.groups.filter(name__contains=y.name).delete()
+                            print(f'Removing: {profile.handle} from Organization: {y.name} ')
+
+                        user_access_repos = get_repo(handle, '/repos', (handle, access_token), is_user=True)
+                        # Question around user repo acccess if we can't get user repos, should we assume all repos are no longer available in the platform?
+
+                        if 'message' in user_access_repos:
+                            print(user_access_repos['message'])
+                            return lsynced
+
+                        current_user_repos = []
+                        for y in user_access_repos:
+                            current_user_repos.append(y['name'])
+
+                        remove_user_repos_names = [x for x in profile.repos.all() if
+                                                   x.name not in current_user_repos]
+
+                        remove_user_repos = Repo.objects.filter(name__in=remove_user_repos_names,
+                                                                profile__handle=handle)
+                        for y in remove_user_repos:
+                            profile.repos.remove(y)
+
+                        lsynced.append(handle)
+                    else:
+                        return lsynced
+
+                    members_to_sync = []
+                    if profile.organizations is None:
+                        print("no organizations to sync")
+                        return []
+
+                    for org in profile.organizations:
+                        try:
+                            if org in orgs_synced:
+                                print(f'{org} has been synced already')
+                                continue
+
+                            orgs_synced.append(org)
+                            db_org = Organization.objects.get_or_create(name=org)[0]
+
+                            print(f'Syncing Organization: {db_org.name}')
+                            profile.profile_organizations.add(db_org)
+                            org_members = get_organization(
+                                db_org.name,
+                                '/members',
+                                (handle, access_token)
+                            )
+
+                            if 'message' in org_members:
+                                print(org_members['message'])
+                                continue
+
+                            for member in org_members:
+
+                                try:
+                                    membership = get_organization(
+                                        db_org.name,
+                                        f'/memberships/{member["login"]}',
+                                        (handle, access_token)
+                                    )
+                                    if 'message' in membership:
+                                        print(membership['message'])
+                                        continue
+                                    role = membership['role'] if 'role' in membership else "member"
+                                    db_group = Group.objects.get_or_create(name=f'{db_org.name}-role-{role}')[0]
+                                    db_org.groups.add(db_group)
+                                    member_profile_obj = Profile.objects.get(
+                                        handle=member['login'],
+                                        user__is_active=True
+                                    )
+                                    member_profile_obj.user.groups.add(db_group)
+                                    members_to_sync.append(member['login'])
+                                except Exception as e:
+                                    print(f'An exception happened in the Organization Loop: handle {member["login"]} {e}')
+                                    continue
+
+                            org_repos = get_organization(
+                                db_org.name,
+                                '/repos',
+                                (handle, access_token)
+                            )
+
+
+                            if 'message' in org_repos:
+                                print(org_repos['message'])
+                                continue
+
+                            for repo in org_repos:
+                                db_repo = Repo.objects.get_or_create(name=repo['name'])[0]
+                                db_org.repos.add(db_repo)
+                                print(f'Syncing Repo: {db_repo.name}')
+                                repo_collabs = get_repo(
+                                    repo['full_name'],
+                                    '/collaborators',
+                                    (handle, access_token)
+                                )
+                                if 'message' in repo_collabs:
+                                    print(repo_collabs['message'])
+                                    continue
+
+                                for collaborator in repo_collabs:
+
+                                    if collaborator['permissions']['admin']:
+                                        permission = "admin"
+                                    elif collaborator['permissions']['push']:
+                                        permission = "write"
+                                    elif collaborator['permissions']['pull']:
+                                        permission = "pull"
+                                    else:
+                                        permission = "none"
+
+                                    db_group = Group.objects.get_or_create(
+                                        name=f'{db_org.name}-repo-{repo["name"]}-{permission}')[0]
+                                    db_org.groups.add(db_group)
+
+                                    try:
+                                        member_user_profile = Profile.objects.get(handle=collaborator['login'],
+                                                                                  user__is_active=True)
+                                        member_user_profile.user.groups.add(db_group)
+                                        member_user_profile.repos.add(db_repo)
+                                        if collaborator['login'] not in members_to_sync or \
+                                            collaborator['login'] not in lsynced:
+                                            members_to_sync.append(collaborator['login'])
+                                    except Exception as e:
+                                        print(f'An exception happened in the Collaborators sync Loop: handle: '
+                                              f'{collaborator["login"]} {e}')
+
+                            for x in members_to_sync:
+                                try:
+                                    lsynced = lsynced + recursive_sync(lsynced, x)
+                                except Exception as e:
+                                    print(f'An exception happened in the Members sync Loop: handle: {handle} {e}')
+                        except Exception as e:
+                            print(f'An exception happened in the Organization Loop: handle {handle} {e}')
+                except Exception as exc:
+                    print(f'Exception occurred inside recursive_sync: {exc}')
+
+                return lsynced
+
+            for profile in all_users:
+                try:
+                    if profile.handle is not None:
+                        synced = recursive_sync(synced, profile.handle)
+                except ValueError as loop_exc:
+                    print(f'Error syncing user id:{profile.user.id}')
+                    print(loop_exc)
+                except Exception as e:
+                    print(e)
+
+            print("Sync Completed")
+        except ValueError as e:
+            print(e)

app/app/settings.py

@@ -531,7 +531,7 @@
 SOCIAL_AUTH_GITHUB_SECRET = GITHUB_CLIENT_SECRET
 SOCIAL_AUTH_POSTGRES_JSONFIELD = True
 SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = ['username', 'first_name', 'last_name', 'email']
-SOCIAL_AUTH_GITHUB_SCOPE = ['read:public_repo', 'read:user', 'user:email', ]
+SOCIAL_AUTH_GITHUB_SCOPE = ['read:public_repo', 'read:user', 'user:email', 'read:org', 'repo']
 SOCIAL_AUTH_SANITIZE_REDIRECTS = True
 
 SOCIAL_AUTH_PIPELINE = (

app/app/urls.py

@@ -122,6 +122,7 @@
         dashboard.views.social_contribution_email,
         name='social_contribution_email'
     ),
+    url(r'^api/v0.1/org_perms', dashboard.views.org_perms, name='org_perms'),
     url(r'^api/v0.1/bulk_invite', dashboard.views.bulk_invite, name='bulk_invite'),
     url(r'^api/v0.1/bountydocument', dashboard.views.bounty_upload_nda, name='bounty_upload_nda'),
     url(r'^api/v0.1/faucet/save/?', faucet.views.save_faucet, name='save_faucet'),
@@ -518,6 +519,7 @@
     re_path(r'^settings/account/?', marketing.views.account_settings, name='account_settings'),
     re_path(r'^settings/tokens/?', marketing.views.token_settings, name='token_settings'),
     re_path(r'^settings/job/?', marketing.views.job_settings, name='job_settings'),
+    re_path(r'^settings/organizations/?', marketing.views.org_settings, name='org_settings'),
     re_path(r'^settings/(.*)?', marketing.views.email_settings, name='settings'),
 
     # marketing views

app/app/utils.py

@@ -183,7 +183,7 @@ def get_upload_filename(instance, filename):
 
 
 def sync_profile(handle, user=None, hide_profile=True):
-    from dashboard.models import Profile
+    from dashboard.models import Profile, Organization
     handle = handle.strip().replace('@', '').lower()
     data = get_user(handle)
     email = ''
@@ -207,9 +207,10 @@ def sync_profile(handle, user=None, hide_profile=True):
     # store the org info in postgres
     try:
         profile, created = Profile.objects.update_or_create(handle=handle, defaults=defaults)
-        print("Profile:", profile, "- created" if created else "- updated")
-        orgs = get_user(handle, '/orgs')
+        access_token = profile.user.social_auth.filter(provider='github').latest('pk').access_token
+        orgs = get_user(handle, '', scope='orgs', auth=(profile.handle, access_token))
         profile.organizations = [ele['login'] for ele in orgs]
+        print("Profile:", profile, "- created" if created else "- updated")
         keywords = []
         for repo in profile.repos_data_lite:
             language = repo.get('language') if repo.get('language') else ''

app/dashboard/migrations/0054_auto_20190930_1525.py

@@ -0,0 +1,51 @@
+# Generated by Django 2.2.4 on 2019-09-30 15:25
+
+from django.db import migrations, models
+import economy.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('auth', '0011_update_proxy_permissions'),
+        ('dashboard', '0053_auto_20190920_1816'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Repo',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_on', models.DateTimeField(db_index=True, default=economy.models.get_time)),
+                ('modified_on', models.DateTimeField(default=economy.models.get_time)),
+                ('name', models.CharField(max_length=255)),
+            ],
+            options={
+                'ordering': ('name',),
+            },
+        ),
+        migrations.CreateModel(
+            name='Organization',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_on', models.DateTimeField(db_index=True, default=economy.models.get_time)),
+                ('modified_on', models.DateTimeField(default=economy.models.get_time)),
+                ('name', models.CharField(max_length=255)),
+                ('groups', models.ManyToManyField(blank=True, to='auth.Group')),
+                ('repos', models.ManyToManyField(blank=True, to='dashboard.Repo')),
+            ],
+            options={
+                'ordering': ('name',),
+            },
+        ),
+        migrations.AddField(
+            model_name='profile',
+            name='profile_organizations',
+            field=models.ManyToManyField(blank=True, to='dashboard.Organization'),
+        ),
+        migrations.AddField(
+            model_name='profile',
+            name='repos',
+            field=models.ManyToManyField(blank=True, to='dashboard.Repo'),
+        ),
+    ]

app/dashboard/migrations/0059_merge_20191022_1603.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-22 16:03
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dashboard', '0054_auto_20190930_1525'),
+        ('dashboard', '0058_auto_20191017_1723'),
+    ]
+
+    operations = [
+    ]

app/dashboard/models.py

@@ -2153,6 +2153,28 @@ def hidden(self):
         return self.filter(hide_profile=True)
 
 
+class Repo(SuperModel):
+    name = models.CharField(max_length=255)
+
+    class Meta:
+        ordering = ('name',)
+
+    def __str__(self):
+        return self.name
+
+
+class Organization(SuperModel):
+    name = models.CharField(max_length=255)
+    groups = models.ManyToManyField('auth.Group', blank=True)
+    repos = models.ManyToManyField(Repo, blank=True)
+
+    class Meta:
+        ordering = ('name',)
+
+    def __str__(self):
+        return self.name
+
+
 class HackathonRegistration(SuperModel):
     """Defines the Hackthon profiles registrations"""
     name = models.CharField(max_length=255, help_text='Hackathon slug')
@@ -2170,10 +2192,6 @@ class HackathonRegistration(SuperModel):
         on_delete=models.CASCADE,
         help_text='User profile'
     )
-
-    class Meta:
-        ordering = ('name',)
-
     def __str__(self):
         return f"Name: {self.name}; Hackathon: {self.hackathon}; Referer: {self.referer}; Registrant: {self.registrant}"
 
@@ -2226,6 +2244,8 @@ class Profile(SuperModel):
     )
     keywords = ArrayField(models.CharField(max_length=200), blank=True, default=list)
     organizations = ArrayField(models.CharField(max_length=200), blank=True, default=list)
+    profile_organizations = models.ManyToManyField(Organization, blank=True)
+    repos = models.ManyToManyField(Repo, blank=True)
     form_submission_records = JSONField(default=list, blank=True)
     max_num_issues_start_work = models.IntegerField(default=3)
     preferred_payout_address = models.CharField(max_length=255, default='', blank=True)
@@ -2273,10 +2293,8 @@ class Profile(SuperModel):
     rank_coder = models.IntegerField(default=0)
     referrer = models.ForeignKey('dashboard.Profile', related_name='referred', on_delete=models.CASCADE, null=True, db_index=True, blank=True)
 
-
     objects = ProfileQuerySet.as_manager()
 
-
     @property
     def quest_level(self):
         return self.quest_attempts.filter(success=True).distinct('quest').count() + 1
@@ -2689,7 +2707,6 @@ def calc_activity_level(self):
         return "Low"
 
 
-
     def calc_longest_streak(self):
         """ Determines the longest streak, in workdays, of this user