✨分支项目支持指定扫描路径

server/projects/main/apps/codeproj/api_filters/base.py

@@ -47,6 +47,7 @@ class ApiProjectFilter(filters.FilterSet):
     scm_url = filters.CharFilter(label="代码库#分支地址", help_text="会根据仓库地址及分支进行匹配，代码库与分支使用#间隔",
                                  method="scm_url_branch_filter")
     branch = filters.CharFilter(label="代码库分支", help_text="代码库分支筛选", lookup_expr="exact")
+    scan_path = filters.CharFilter(label="扫描路径", help_text="扫描路径筛选", lookup_expr="icontains")
     scm_url__exact = filters.CharFilter(label="代码库地址精准匹配", help_text="会根据仓库地址进行匹配",
                                         method="scm_url_filter")
     created_time__lte = filters.DateTimeFilter(field_name="created_time",
@@ -87,7 +88,7 @@ def scm_url_filter(self, queryset, name, value):
     class Meta:
         model = models.Project
         fields = ["scm_url", "scm_url__exact", "scan_scheme__name", "scan_scheme__default_flag",
-                  "scan_scheme_id", "branch", "created_time__gte", "created_time__lte"]
+                  "scan_scheme_id", "branch", "created_time__gte", "created_time__lte", "scan_path"]
 
 
 class ScanSchemeFilter(filters.FilterSet):
@@ -194,18 +195,22 @@ class ProjectFilter(filters.FilterSet):
     """项目筛选
     """
     branch = filters.CharFilter(help_text="分支名称")
+    scan_path = filters.CharFilter(help_text="扫描路径筛选", lookup_expr="icontains")
     scan_scheme = filters.NumberFilter(help_text="扫描方案编号")
     scan_scheme__name = filters.CharFilter(help_text="扫描方案名称")
     scan_scheme__status = filters.NumberFilter(help_text="扫描方案状态，1为活跃，2为废弃")
     branch_or_scheme = filters.CharFilter(label="分支名称/扫描方案名称", help_text="分支名称/扫描方案名称",
-                                          method="branch_or_scheme_filter")
+                                          method="branch_or_scheme_or_path_filter")
 
-    def branch_or_scheme_filter(self, queryset, name, value):
-        return queryset.filter(Q(branch__icontains=value) | Q(scan_scheme__name__icontains=value))
+    def branch_or_scheme_or_path_filter(self, queryset, name, value):
+        return queryset.filter(
+            Q(branch__icontains=value) |
+            Q(scan_scheme__name__icontains=value) |
+            Q(scan_path__icontains=value))
 
     class Meta:
         model = models.Project
-        fields = ["branch", "scan_scheme", "branch_or_scheme", "scan_scheme__status", "status",
+        fields = ["branch", "scan_scheme", "scan_path", "branch_or_scheme", "scan_scheme__status", "status",
                   "scan_scheme__default_flag", "scan_scheme__name"]
 
 

server/projects/main/apps/codeproj/core/base.py

@@ -55,20 +55,22 @@ def create_local_scan(project, creator, scan_data, created_from="codedog_client"
     }
     job_context.update(**scan_data)
     job_manager = JobManager(project)
-    logger.info("开始初始化任务，任务参数: %s" % json.dumps(job_context, indent=2))
+    logger.info("[Project: %s] 开始初始化任务，任务参数: %s" % (project.id, json.dumps(job_context)))
     scan_type = job_manager.get_scan_type(scan_data, job_context)
     job = job_manager.initialize_job(
         force_create=job_context["force_create"],
         creator=creator, created_from=created_from,
         client_flag=True
     )
+    logger.info("[Project: %s] 初始化任务完成，任务编号: %s" % (project.id, job.id))
     try:
         job.context = job_context
         job.save()
         job_manager.create_waiting_scan_on_analysis_server(job, scan_type)
         task_infos = None
         if task_names:
             task_infos = job_manager.init_tasks(job, task_names)
+        logger.info("[Project: %s] 创建任务完成: %s，关联扫描编号：%s" % (project.id, job.id, job.scan_id))
         return job.id, job.scan_id, task_infos
     except Exception as err:
         if isinstance(err, CDErrorBase):

server/projects/main/apps/codeproj/core/jobmgr.py

@@ -8,14 +8,17 @@
 """
 codeproj - job core
 """
+# 原生 import
 import copy
 import logging
 from datetime import datetime, timedelta
 
+# 第三方 import
 from django.conf import settings
 from django.db import transaction
 from django.utils import timezone
 
+# 项目内 import
 from apps.codeproj import models
 from apps.codeproj.core.projmgr import ScanSchemeManager
 from apps.codeproj.core.scmmgr import ScmClientManager
@@ -210,6 +213,7 @@ def get_job_basic_confs(self, job_context, scan_scheme=None):
             job_context.update({
                 "project_id": self._project.id,
                 "repo_id": self._project.repo.id,
+                "scan_path": self._project.scan_path,
                 "team_name": project_team.name if project_team else None,
                 "org_sid": organization.org_sid if organization else None,
                 "scm_type": self._project.scm_type,
@@ -370,7 +374,7 @@ def get_codelint_task_confs(self, job_context, scm_last_revision=None, **kwargs)
         else:
             scan_scheme_languages = [l.name for l in scan_scheme.languages.all()]
         if not scan_scheme_languages:
-            raise ClientError(errcode.E_CLIENT_CONFIG_ERROR, '配置错误，请在项目配置中设置项目语言。')
+            raise ClientError(errcode.E_USER_CONFIG_LANG_ERROR, '配置错误，请在项目配置中设置项目语言。')
         task_basic_params = {
             "scan_languages": scan_scheme_languages,
             "pre_cmd": lint_setting.pre_cmd,
@@ -393,7 +397,7 @@ def get_codelint_task_confs(self, job_context, scm_last_revision=None, **kwargs)
             return self._get_codelint_task_confs_by_checkprofile(
                 checkprofile, task_basic_params)
         else:
-            raise ClientError(errcode.E_CLIENT_CONFIG_ERROR, '未绑定扫描规则集，请在项目配置-扫描设置中绑定。')
+            raise ClientError(errcode.E_USER_CONFIG_CODELINT_PKG_ERROR, '未配置扫描规则，请在"扫描方案-代码扫描"中添加')
 
     def get_codemetric_task_confs(self, job_context, scm_last_revision=None, **kwargs):
         """
@@ -436,7 +440,7 @@ def get_codemetric_task_confs(self, job_context, scm_last_revision=None, **kwarg
         # 增加cpd task 重复代码扫描
         if metric_setting.dup_scan_enabled:
             if not scan_scheme_languages:
-                raise ClientError(errcode.E_CLIENT_CONFIG_ERROR, '配置错误，请在项目配置中设置项目语言。')
+                raise ClientError(errcode.E_USER_CONFIG_LANG_ERROR, '配置错误，请在项目配置中设置项目语言。')
             last_dup_scan = models.CodeMetricDupInfo.objects.filter(project=self._project).first()
             if not scm_last_revision and self._project:
                 scm_last_revision = last_dup_scan.scan_revision if last_dup_scan else self._project.scm_initial_revision
@@ -464,7 +468,7 @@ def get_codemetric_task_confs(self, job_context, scm_last_revision=None, **kwarg
         # 增加lizard task 圈复杂度
         if metric_setting.cc_scan_enabled:
             if not scan_scheme_languages:
-                raise ClientError(errcode.E_CLIENT_CONFIG_ERROR, '配置错误，请在项目配置中设置项目语言。')
+                raise ClientError(errcode.E_USER_CONFIG_LANG_ERROR, '配置错误，请在项目配置中设置项目语言。')
 
             last_cc_scan = models.CodeMetricCCInfo.objects.filter(project=self._project).first()
             if not scm_last_revision and self._project:
@@ -504,6 +508,8 @@ def get_codemetric_task_confs(self, job_context, scm_last_revision=None, **kwarg
                 "task_params": task_params,
                 "tag": tag
             })
+
+        # Todo：支持组件分析
         return tasks
 
     def get_job_confs(self, job_context=None, last_revision=None, **kwargs):
@@ -542,7 +548,7 @@ def get_job_confs(self, job_context=None, last_revision=None, **kwargs):
             metric_tasks = self.get_codemetric_task_confs(job_context, last_revision,
                                                           scan_scheme=kwargs.get("scan_scheme"))
             if not lint_tasks and not metric_tasks:
-                raise ClientError(errcode.E_CLIENT_CONFIG_ERROR, '配置错误，请在设置中启用代码检查或代码度量功能')
+                raise ClientError(errcode.E_USER_CONFIG_NO_LINT_OR_METRIC, '配置错误，请在设置中启用代码检查或代码度量功能')
             tasks = lint_tasks + metric_tasks
         except Exception as e:
             if isinstance(e, CDErrorBase):
@@ -598,7 +604,7 @@ def _get_task_confs(self, job_context, last_revision=None):
             logger.info("%s开始获取codemetric参数" % self._log_prefix)
             metric_tasks = self.get_codemetric_task_confs(job_context, last_revision)
             if not lint_tasks and not metric_tasks:
-                raise ClientError(errcode.E_CLIENT_CONFIG_ERROR, '配置错误，请在设置中启用代码检查或代码度量功能')
+                raise ClientError(errcode.E_USER_CONFIG_NO_LINT_OR_METRIC, '配置错误，请在设置中启用代码检查或代码度量功能')
             tasks = lint_tasks + metric_tasks
         except Exception as e:
             if isinstance(e, CDErrorBase):

server/projects/main/apps/codeproj/core/projmgr.py

@@ -8,19 +8,22 @@
 """
 codeproj - project core
 """
+# 原生 import
 import json
 import logging
 import uuid
 
+# 第三方 import
 from django.db.models import Q
 from django.db.utils import IntegrityError
 from django.utils.timezone import now
 
-from apps.authen.models import Organization, ScmAuth, ScmAccount
+# 项目内 import
+from apps.authen.models import Organization, ScmAccount, ScmAuth
 from apps.codeproj import models
-from apps.scan_conf.core import add_checkrules_to_checkpackage, CheckProfileManager, CheckRuleManager
+from apps.scan_conf.core import CheckProfileManager, CheckRuleManager, add_checkrules_to_checkpackage
 from apps.scan_conf.models import CheckProfile, PackageMap
-from util.exceptions import ServerConfigError, ServerOperationError, RepositoryCreateError, errcode, ProjectCreateError
+from util.exceptions import ProjectCreateError, RepositoryCreateError, ServerConfigError, ServerOperationError, errcode
 from util.operationrecord import OperationRecordHandler
 from util.scm import ScmClient
 from util.webclients import AnalyseClient
@@ -771,8 +774,10 @@ def update_scheme_metric_setting(cls, scan_scheme, user=None, **metric_setting_d
         metric_setting.dup_issue_limit = metric_setting_data.get("dup_issue_limit", metric_setting.dup_issue_limit)
         metric_setting.cloc_scan_enabled = metric_setting_data.get("cloc_scan_enabled",
                                                                    metric_setting.cloc_scan_enabled)
-        metric_setting.core_file_path = metric_setting_data.get("core_file_path", metric_setting.core_file_path)
-        metric_setting.file_mon_path = metric_setting_data.get("file_mon_path", metric_setting.file_mon_path)
+        metric_setting.core_file_path = metric_setting_data.get("core_file_path",
+                                                                metric_setting.core_file_path)
+        metric_setting.file_mon_path = metric_setting_data.get("file_mon_path",
+                                                               metric_setting.file_mon_path)
         metric_setting.save(user)
 
     @classmethod
@@ -1025,6 +1030,43 @@ class ProjectManager(object):
     """项目管理
     """
 
+    @classmethod
+    def format_scan_path(cls, scan_path):
+        """格式化scan_path
+        """
+        scan_path = scan_path.strip()
+        if not scan_path or scan_path == "/" or scan_path == "./":
+            scan_path = "/"
+        elif scan_path.startswith("./"):
+            scan_path = scan_path[2:].strip("/")
+        else:
+            scan_path = scan_path.strip("/")
+        return scan_path
+
+    @classmethod
+    def create_project(cls, repo, scan_scheme, branch, scan_path=None, **kwargs):
+        """创建项目
+        """
+        if not scan_path:
+            scan_path = "/"
+        else:
+            scan_path = cls.format_scan_path(scan_path)
+
+        project_key = models.Project.gen_project_key(
+            repo_id=repo.id, scheme_id=scan_scheme.id, branch=branch, scan_path=scan_path)
+        project, created = models.Project.objects.get_or_create(
+            project_key=project_key,
+            defaults={
+                "repo_id": repo.id,
+                "branch": branch,
+                "scan_scheme_id": scan_scheme.id,
+                "scan_path": scan_path,
+                "creator": kwargs.get("creator"),
+                "created_from": kwargs.get("created_from") or models.Project.CreatedFromEnum.WEB,
+                "refer_project": kwargs.get("refer_project"),
+            })
+        return project, created
+
     @classmethod
     def create_project_on_analysis_server(cls, project, user):
         """在Analysis服务器创建项目
@@ -1034,6 +1076,7 @@ def create_project_on_analysis_server(cls, project, user):
                 "id": project.id,
                 "repo_id": project.repo_id,
                 "scan_scheme_id": project.scan_scheme_id,
+                "scan_path": project.scan_path,
                 "creator": user.username,
                 "scm_type": project.scm_type,
                 "scm_url": project.scm_url,
@@ -1056,9 +1099,22 @@ def delete_project(cls, repo_id, project_id, user):
         deleted_time = now()
         logger.info("[User: %s] 在 %s 删除了 %s 分支项目" % (user.username, deleted_time, project))
         project.branch = ("deleted by %s(%s)" % (user.username, deleted_time))[:198]
-        project.remark = json.dumps({"branch": old_branch})
+        project.status = models.Project.StatusEnum.DISACTIVE
+        project.update_remark({"branch": old_branch})
         project.save()
         project.delete(user=user)
         OperationRecordHandler.add_project_operation_record(
             project, "删除分支项目", user.username, message="删除分支项目: %s" % project_str
         )
+
+    @classmethod
+    def filter_disactive_projects(cls, queries):
+        """筛选失活项目
+        """
+        return models.Project.objects.filter(status=models.Project.StatusEnum.DISACTIVE).filter(**queries)
+
+    @classmethod
+    def filter_deleted_projects(cls, queries):
+        """筛选已删除的项目
+        """
+        return models.Project.everything.filter(deleted_time__isnull=False).filter(**queries)

server/projects/main/apps/codeproj/management/commands/refresh_project_key.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021-2022 THL A29 Limited
+#
+# This source code file is made available under MIT License
+# See LICENSE for details
+# ==============================================================================
+
+"""刷新项目key值
+"""
+import logging
+
+# 第三方
+from django.core.management.base import BaseCommand
+
+# 项目内
+from apps.codeproj.models import Project
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "refresh project key"
+
+    def handle(self, *args, **options):
+        ps = Project.everything.filter(project_key__isnull=True)
+        num = ps.count()
+        project_list = []
+        count = 0
+        logger.info("Start refresh project num: %s" % num)
+        for p in ps:
+            count += 1
+            p.scan_path = "/" if not p.scan_path else p.scan_path
+            p.project_key = p.gen_project_key(p.repo_id, p.scan_scheme_id, p.branch, p.scan_path)
+            project_list.append(p)
+            if len(project_list) == 1000:
+                logger.info("Refresh project num: %s/%s" % (count, num))
+                Project.objects.bulk_update(project_list, ["scan_path", "project_key"])
+                project_list = []
+        if project_list:
+            Project.objects.bulk_update(project_list, ["scan_path", "project_key"])
+        logger.info("Finish refresh project num: %s/%s" % (count, num))

server/projects/main/apps/codeproj/migrations/0008_auto_20220803_2054.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+#  Copyright (c) 2021-2022 THL A29 Limited
+#  #
+#  This source code file is made available under MIT License
+#  See LICENSE for details
+#  ==============================================================================
+# Generated by Django 3.1.14 on 2022-08-03 12:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('codeproj', '0007_auto_20220617_1516'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='baseproject',
+            name='project_key',
+            field=models.CharField(blank=True, help_text='项目Key值', max_length=64, null=True, unique=True),
+        ),
+        migrations.AddField(
+            model_name='baseproject',
+            name='scan_path',
+            field=models.CharField(blank=True, help_text='扫描路径', max_length=512, null=True),
+        ),
+        migrations.AlterUniqueTogether(
+            name='baseproject',
+            unique_together=set(),
+        ),
+        migrations.AlterIndexTogether(
+            name='baseproject',
+            index_together={('repo', 'scan_scheme', 'branch')},
+        ),
+    ]