From 9f7acf6d2b0008107d9bbdf55ee71c384967d554 Mon Sep 17 00:00:00 2001 From: Paul Przybyszewski Date: Tue, 26 Jul 2022 20:09:45 +0200 Subject: [PATCH 1/2] codebase: remove some redundant code in config.py and github.py --- cogs/ecosystem/config.py | 10 +++------- lib/api/github.py | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cogs/ecosystem/config.py b/cogs/ecosystem/config.py index 3655be37..76475672 100644 --- a/cogs/ecosystem/config.py +++ b/cogs/ecosystem/config.py @@ -71,12 +71,8 @@ async def config_show_command_group(self, ctx: GitBotContext) -> None: await ctx.error(ctx.l.generic.nonexistent.qa) return lang: str = ctx.fmt('accessibility list locale', f'`{ctx.l.meta.localized_name.capitalize()}`') - user_str: str = ctx.fmt('qa list user', (f'[`{user["user"]}`](https://github.com/{user["user"]})' - if 'user' in user else f'`{ctx.l.config.show.base.item_not_set}`')) - org: str = ctx.fmt('qa list org', (f'[`{user["org"]}`](https://github.com/{user["org"]})' - if 'org' in user else f'`{ctx.l.config.show.base.item_not_set}`')) - repo: str = ctx.fmt('qa list repo', (f'[`{user["repo"]}`](https://github.com/{user["repo"]})' - if 'repo' in user else f'`{ctx.l.config.show.base.item_not_set}`')) + user_str, org, repo = (ctx.fmt(f'qa list {item}', f'[`{item}`](https://github.com/{item})' if item in user else + f'`{ctx.l.config.show.base.item_not_set}`') for item in ('user', 'org', 'repo')) accessibility: list = ctx.l.config.show.base.accessibility.heading + '\n' + '\n'.join([lang]) qa: list = ctx.l.config.show.base.qa.heading + '\n' + '\n'.join([user_str, org, repo]) guild_str: str = '' @@ -90,7 +86,7 @@ async def config_show_command_group(self, ctx: GitBotContext) -> None: ac: AutomaticConversionSettings = Mgr.env.autoconv_default else: ac: AutomaticConversionSettings = {k: (v if k not in (_ac := guild.get('autoconv', {})) - else _ac[k]) for k, v in Mgr.env.autoconv_default.items()} + else _ac[k]) for k, v in Mgr.env.autoconv_default.items()} codeblock: str = ctx.fmt('codeblock', f'`{ctx.l.enum.generic.switch[str(ac["codeblock"])]}`') lines: str = ctx.fmt('gh_lines', diff --git a/lib/api/github.py b/lib/api/github.py index 0dbb6475..c8dd9065 100644 --- a/lib/api/github.py +++ b/lib/api/github.py @@ -44,7 +44,7 @@ class GitHubAPI: """ def __init__(self, tokens: tuple, requester: str): - requester: str = requester + '; Python {v.major}.{v.minor}.{v.micro}'.format(v=version_info) + requester += '; Python {v.major}.{v.minor}.{v.micro}'.format(v=version_info) self.__tokens: tuple = tokens self.__token_cycle: cycle = cycle(t for t in self.__tokens if t is not None) self.queries: DirProxy = DirProxy('./resources/queries/', ('.gql', '.graphql')) From 0d792a0e5a02e1fe90820aa338c481792fc1ea8b Mon Sep 17 00:00:00 2001 From: Paul Przybyszewski Date: Tue, 26 Jul 2022 21:38:51 +0200 Subject: [PATCH 2/2] codebase: add the ability to ignore files/dirs in the loc command --- cogs/github/other/loc.py | 49 ++++++++++++++++++++++++++++---- lib/api/github.py | 2 +- lib/manager.py | 13 +++++++-- lib/typehints/__init__.py | 1 + lib/typehints/gitbot_dot_json.py | 14 +++++++++ resources/locale/en.locale.json | 8 +++++- 6 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 lib/typehints/gitbot_dot_json.py diff --git a/cogs/github/other/loc.py b/cogs/github/other/loc.py index eb7ec780..d994fbd6 100644 --- a/cogs/github/other/loc.py +++ b/cogs/github/other/loc.py @@ -3,6 +3,7 @@ import json import aiofiles import shutil +import fnmatch import subprocess from discord.ext import commands from typing import Optional @@ -16,6 +17,9 @@ class LinesOfCode(commands.Cog): + # I know that using "perl" alone is insecure, but it will only be used in Windows dev environments + __perl_command_line__: str = '/bin/perl' if os.name != 'nt' else 'perl' + def __init__(self, bot: commands.Bot): self.bot: commands.Bot = bot @@ -35,11 +39,13 @@ async def lines_of_code_command(self, ctx: GitBotContext, repo: GitHubRepository if not r: await ctx.error(ctx.l.generic.nonexistent.repo.base) return - processed: Optional[dict] = await self.process_repo(ctx, repo) + processed: Optional[tuple[dict, int | None]] | dict = await self.process_repo(ctx, repo) if not processed: await ctx.error(ctx.l.loc.file_too_big) return title: str = ctx.fmt('title', f'`{repo}`') + count: int | None = processed[1] + processed = processed[0] embed: GitBotEmbed = GitBotEmbed( color=0x00a6ff, title=title, @@ -52,11 +58,31 @@ async def lines_of_code_command(self, ctx: GitBotContext, repo: GitHubRepository + f'**{ctx.l.loc.stats.comments}:** {processed["SUM"]["comment"]}\n' + f'**{ctx.l.loc.stats.detailed}:**\n' + await self.prepare_result_sheet(processed)), - footer=ctx.l.loc.footer + footer=ctx.l.loc.footer.credit if not count + else (ctx.fmt('footer with_count plural', count) if count > 1 else ctx.fmt('footer with_count singular')), ) await ctx.send(embed=embed) - async def process_repo(self, ctx: GitBotContext, repo: GitHubRepository) -> Optional[dict]: + @staticmethod + def remove_matches(directory: str, pattern: str) -> int: + Mgr.debug(f'Removing files matching pattern "{pattern}" from directory "{directory}"') + c_removed: int = 0 + for root, dirs, files in os.walk(directory): + for f in files: + if fnmatch.fnmatch(f, pattern): + Mgr.debug(f'Removing file "{f}"') + c_removed += 1 + os.remove(os.path.join(root, f)) + for d in dirs: + if fnmatch.fnmatch(d, pattern): + Mgr.debug(f'Removing directory "{d}"') + c_removed += 1 + shutil.rmtree(os.path.join(root, d)) + Mgr.debug(f'Removed {c_removed} entries.') + return c_removed + + @staticmethod + async def process_repo(ctx: GitBotContext, repo: GitHubRepository) -> Optional[tuple[dict, int | None]]: if (not ctx.__nocache__) and (cached := Mgr.loc_cache.get(repo := repo.lower())): return cached tmp_zip_path: str = f'./tmp/{ctx.message.id}.zip' @@ -70,12 +96,22 @@ async def process_repo(self, ctx: GitBotContext, repo: GitHubRepository) -> Opti async with aiofiles.open(tmp_zip_path, 'wb') as fp: await fp.write(files) await Mgr.unzip_file(tmp_zip_path, tmp_dir_path) - output: dict = json.loads(subprocess.check_output(['/bin/perl', 'cloc.pl', '--json', tmp_dir_path])) + c_removed: int = 0 + cfg: dict | None = await Mgr.get_repo_gitbot_config(repo) + if cfg and cfg.get('loc'): + if isinstance(cfg['loc'], dict) and (ignore := cfg['loc'].get('ignore')): + if isinstance(ignore, str): + ignore = [ignore] + c_removed: int = 0 + for pattern in ignore: + c_removed += LinesOfCode.remove_matches(tmp_dir_path, pattern) + output: dict = json.loads(subprocess.check_output([LinesOfCode.__perl_command_line__, 'cloc.pl', + '--json', tmp_dir_path])) except subprocess.CalledProcessError as e: Mgr.debug(f'the CLOC script failed with exit code {e.returncode}') else: Mgr.loc_cache[repo] = output - return output + return output, c_removed finally: try: shutil.rmtree(tmp_dir_path) @@ -83,7 +119,8 @@ async def process_repo(self, ctx: GitBotContext, repo: GitHubRepository) -> Opti except FileNotFoundError: pass - async def prepare_result_sheet(self, data: dict) -> str: + @staticmethod + async def prepare_result_sheet(data: dict) -> str: result: str = '```py\n{}```' threshold: int = 15 for k, v in data.items(): diff --git a/lib/api/github.py b/lib/api/github.py index c8dd9065..f5699667 100644 --- a/lib/api/github.py +++ b/lib/api/github.py @@ -107,7 +107,7 @@ async def get_repo_files(self, repo: GitHubRepository) -> list[dict]: return [] @normalize_repository - async def get_tree_file(self, repo: GitHubRepository, path: str): + async def get_tree_file(self, repo: GitHubRepository, path: str) -> dict | list: if repo.count('/') != 1: return [] if path[0] == '/': diff --git a/lib/manager.py b/lib/manager.py index b3ba558f..28027b39 100644 --- a/lib/manager.py +++ b/lib/manager.py @@ -15,6 +15,7 @@ import ast import json import dotenv +import base64 import discord import zipfile import os.path @@ -46,7 +47,8 @@ from typing import Optional, Callable, Any, Reversible, Iterable, Type, TYPE_CHECKING, Generator if TYPE_CHECKING: from lib.structs.discord.context import GitBotContext -from lib.typehints import DictSequence, AnyDict, Identity, GitBotGuild, AutomaticConversionSettings, LocaleName, ReleaseFeedItemMention + from lib.api.github import GitHubAPI +from lib.typehints import DictSequence, AnyDict, Identity, GitBotGuild, AutomaticConversionSettings, LocaleName, ReleaseFeedItemMention, GitBotDotJSON class Manager: @@ -60,7 +62,7 @@ class Manager: def __init__(self, github): self.lib_root: str = os.path.dirname(os.path.abspath(__file__)) self.root_directory: str = self.lib_root[:self.lib_root.rindex(os.sep)] - self.git = github + self.git: 'GitHubAPI' = github self.ses: aiohttp.ClientSession = self.git.ses self._prepare_env() self.bot_dev_name: str = f'gitbot ({"production" if self.env.production else "preview"})' @@ -83,6 +85,13 @@ def __init__(self, github): self.__fix_missing_locales() self.__preprocess_locale_emojis() + async def get_repo_gitbot_config(self, repo: str) -> GitBotDotJSON | None: + gh_res: dict | None = await self.git.get_tree_file(repo, '.gitbot.json') + if not gh_res: + return + if gh_res['encoding'] == 'base64': + return json.loads(base64.decodebytes(bytes(gh_res['content'].encode('utf-8'))).decode('utf-8')) + def get_current_commit(self, short: bool = True) -> str: """ Get the current commit hash of the running bot instance. diff --git a/lib/typehints/__init__.py b/lib/typehints/__init__.py index c22ddfc9..3e004a2c 100644 --- a/lib/typehints/__init__.py +++ b/lib/typehints/__init__.py @@ -7,6 +7,7 @@ from lib.typehints.generic import * from lib.typehints.locale.help import * from lib.typehints.db.guild.release_feed import * +from lib.typehints.gitbot_dot_json import * __all__: tuple = ( 'DictSequence', diff --git a/lib/typehints/gitbot_dot_json.py b/lib/typehints/gitbot_dot_json.py new file mode 100644 index 00000000..c0721960 --- /dev/null +++ b/lib/typehints/gitbot_dot_json.py @@ -0,0 +1,14 @@ +from typing import TypedDict + + +class LOCGitBotJSONEntry(TypedDict): + ignore: str | list[str] + + +class ReleaseFeedGitBotJSONEntry(TypedDict): + ignore: str + + +class GitBotDotJSON(TypedDict, total=False): + loc: LOCGitBotJSONEntry + release_feed: ReleaseFeedGitBotJSONEntry diff --git a/resources/locale/en.locale.json b/resources/locale/en.locale.json index b33fce19..09d6b467 100644 --- a/resources/locale/en.locale.json +++ b/resources/locale/en.locale.json @@ -399,7 +399,13 @@ "comments": "Comments", "detailed": "Detailed" }, - "footer": "This command is powered by the CLOC CLI tool." + "footer": { + "with_count": { + "plural": "{0} entries matching .gitbot.json ignore rules were removed.", + "singular": "One entry matching .gitbot.json ignore rules was removed." + }, + "credit": "This command is powered by the CLOC CLI tool." + } }, "commits": { "embed": {