From 169bab9bffb4b987e87448525e59813a658edb76 Mon Sep 17 00:00:00 2001 From: Cyrille Bagard Date: Tue, 16 Jan 2018 00:59:59 +0100 Subject: Written a first usable version. --- board.py | 159 ++++++++++++++++++------------ chrysalide.sh | 4 + code.py | 206 +++++++++++++++++++++++++++++++++++++++ gas.py | 94 +++++++++++------- repository.py | 307 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 675 insertions(+), 95 deletions(-) create mode 100644 code.py create mode 100644 repository.py diff --git a/board.py b/board.py index aeda474..35e5735 100644 --- a/board.py +++ b/board.py @@ -99,8 +99,8 @@ class DynamicFlyer: hmargin = self._width * 0.1 vmargin = self._square * 0.1 - width = self._width - 2 * hmargin - height = self._square - 2 * vmargin + width = self._width - 2 * hmargin - 10 + height = self._square - 2 * vmargin - 10 self._cr.save() @@ -108,37 +108,40 @@ class DynamicFlyer: # Find the max and normalize + min_vol = min(data.values()) max_vol = max(data.values()) + diff_vol = max_vol - min_vol + sx = width / (end - start) - sy = height / max_vol + sy = height / diff_vol + + # Display volumes self._cr.save() self._cr.scale(sx, sy) - # Display volumes - first = min(data.keys()) if created: - self._cr.move_to(first, max_vol) + self._cr.move_to(first - start, diff_vol) else: - self._cr.move_to(0, max_vol) + self._cr.move_to(0, diff_vol) self._cr.line_to(0, max_vol - data[first]) for time in sorted(data.keys()): vol = data[time] - self._cr.line_to(time, max_vol - vol) + self._cr.line_to(time - start, max_vol - vol) last = max(data.keys()) - self._cr.line_to(last, max_vol) + self._cr.line_to(last - start, diff_vol) self._cr.close_path() @@ -146,19 +149,9 @@ class DynamicFlyer: self._cr.fill_preserve() self._cr.set_source_rgba(*color, 1.0) - self._cr.set_line_width(1 / sx) + self._cr.set_line_width(1 / (sx if sx > sy else sy)) self._cr.stroke() - # Display commits - - for time, vol in data.items(): - - self._cr.rectangle(time - 2 / sx, max_vol - vol - 2 / sy, 4 / sx, 4 / sy) - - self._cr.set_source_rgba(*color, 1.0) - self._cr.set_line_width(0.5) - self._cr.fill() - self._cr.restore() # Display the borders @@ -170,26 +163,52 @@ class DynamicFlyer: self._cr.set_source_rgba(*self._forecolor, 0.7) self._cr.stroke() - self._cr.move_to(0, 0) - self._cr.line_to(-4, 10) - self._cr.line_to(4, 10) + self._cr.move_to(0, -10) + self._cr.line_to(-4, 0) + self._cr.line_to(4, 0) self._cr.set_source_rgba(*self._forecolor, 1.0) self._cr.fill() - self._cr.move_to(width, height) - self._cr.line_to(width - 10, height - 4) - self._cr.line_to(width - 10, height + 4) + self._cr.move_to(width + 10, height) + self._cr.line_to(width, height - 4) + self._cr.line_to(width, height + 4) self._cr.set_source_rgba(*self._forecolor, 1.0) self._cr.fill() + # Display some time marks + + unit = width / 12 + + for i in range(11): + + self._cr.move_to((i + 1) * unit, height - 2) + self._cr.line_to((i + 1) * unit, height + 2) + + self._cr.set_source_rgba(*self._forecolor, 1.0) + self._cr.stroke() + + # Display commits + + self._cr.save() + + self._cr.scale(sx, sy) + + for time, vol in data.items(): + + self._cr.rectangle(time - start - 2 / sx, max_vol - vol - 2 / sy, 4 / sx, 4 / sy) + + self._cr.set_source_rgba(*color, 1.0) + self._cr.fill() + self._cr.restore() # Print a caption as explaination + self._cr.restore() + count = len(data.values()) - min_vol = min(data.values()) unit = '' @@ -232,7 +251,7 @@ class DynamicFlyer: self._cr.restore() - def _render_pie(self, title, data, colors): + def _render_pie(self, title, data): """Render a pie chart.""" cx = self._square / 2 @@ -249,10 +268,12 @@ class DynamicFlyer: for k, v in data.items(): - stats.append((v, k)) + val, color = v + + stats.append((val, k, color)) count += 1 - total += v + total += val stats = sorted(stats) @@ -260,33 +281,38 @@ class DynamicFlyer: done = 0 - init_angle = - (stats[0][0] * math.pi) / (2 * total) + init_angle = - (stats[0][0] * 2 * math.pi) / (total * 2) last_angle = init_angle for stat in stats: - angle1 = last_angle + value, _, color = stat - if (done + 1) == count: - angle2 = init_angle + if value > 0: - else: - angle2 = angle1 + (stat[0] * math.pi) / total + angle1 = last_angle - self._cr.move_to(cx, cy) - self._cr.arc(cx, cy, radius, angle1, angle2) - self._cr.line_to(cx, cy) + if (done + 1) == count: + angle2 = init_angle if init_angle != 0 else 2 * math.pi - self._cr.set_source_rgba(*colors[done], 0.5) - self._cr.fill_preserve() + else: + angle2 = angle1 + (value * 2 * math.pi) / total - self._cr.set_source_rgba(*self._backcolor, 1.0) - self._cr.set_line_width(2) - self._cr.stroke() + self._cr.move_to(cx, cy) + self._cr.arc(cx, cy, radius, angle1, angle2) + self._cr.line_to(cx, cy) + + self._cr.set_source_rgba(*color, 0.5) + self._cr.fill_preserve() + + self._cr.set_source_rgba(*self._backcolor, 1.0) + self._cr.set_line_width(2) + self._cr.stroke() + + last_angle = angle2 done += 1 - last_angle = angle2 done = 0 @@ -294,22 +320,27 @@ class DynamicFlyer: for stat in stats: - angle1 = last_angle + value, _, color = stat - if (done + 1) == count: - angle2 = init_angle + if value > 0: - else: - angle2 = angle1 + (stat[0] * math.pi) / total + angle1 = last_angle - self._cr.arc(cx, cy, radius, angle1, angle2) + if (done + 1) == count: + angle2 = init_angle if init_angle != 0 else 2 * math.pi - self._cr.set_source_rgba(*colors[done], 1.0) - self._cr.set_line_width(10) - self._cr.stroke() + else: + angle2 = angle1 + (value * 2 * math.pi) / total + + self._cr.arc(cx, cy, radius, angle1, angle2) + + self._cr.set_source_rgba(*color, 1.0) + self._cr.set_line_width(10) + self._cr.stroke() + + last_angle = angle2 done += 1 - last_angle = angle2 # Printing the title @@ -346,7 +377,9 @@ class DynamicFlyer: for stat in stats: - x_off, y_off, tw, th = self._cr.text_extents(stat[1])[:4] + _, name, _ = stat + + x_off, y_off, tw, th = self._cr.text_extents(name)[:4] if full_width > 0: full_width += self._caption_sep @@ -365,7 +398,9 @@ class DynamicFlyer: for stat in stats: - x_off, y_off, tw, th = self._cr.text_extents(stat[1])[:4] + _, name, color = stat + + x_off, y_off, tw, th = self._cr.text_extents(name)[:4] self._cr.save() @@ -378,10 +413,10 @@ class DynamicFlyer: self._cr.rectangle(0, top, self._caption_width, self._caption_width / 2) - self._cr.set_source_rgba(*colors[done], 0.7) + self._cr.set_source_rgba(*color, 0.7) self._cr.fill_preserve() - self._cr.set_source_rgba(*colors[done], 1.0) + self._cr.set_source_rgba(*color, 1.0) self._cr.set_line_width(1) self._cr.stroke() @@ -391,7 +426,7 @@ class DynamicFlyer: self._cr.set_source_rgba(*self._forecolor, 1.0) - self._cr.show_text(stat[1]) + self._cr.show_text(name) self._cr.restore() @@ -400,12 +435,12 @@ class DynamicFlyer: done += 1 - def render_pie(self, title, data, colors): + def render_pie(self, title, data): """Render a pie chart.""" self._book_room(self._square, self._square) - self._render_pie(title, data, colors) + self._render_pie(title, data) self._cr.restore() diff --git a/chrysalide.sh b/chrysalide.sh index b43716c..d821e4c 100755 --- a/chrysalide.sh +++ b/chrysalide.sh @@ -2,7 +2,11 @@ OUTPUT="2017-2018.png" +[ -d chrysalide ] || git clone http://git.0xdeadc0de.fr/chrysalide.git + python3 ./gas.py \ + --repo "$PWD/chrysalide" \ + --year 2017 \ --header "Chrysalide's Git repository in 2017" \ --footer "Get more info at chrysalide.re" \ --output $OUTPUT diff --git a/code.py b/code.py new file mode 100644 index 0000000..6d90c46 --- /dev/null +++ b/code.py @@ -0,0 +1,206 @@ +# -*- coding: utf-8 -*- + +import os +from metrics.metrics import process + + +class CodeMetrics: + """Handle source code metrics.""" + + + def __init__(self, rootdir): + """Init the code metrics.""" + + self._rootdir = rootdir + + self._languages = {} + + + def __str__(self): + """Display the collected metrics.""" + + desc = '' + + for l in self._languages: + + desc += l + ': ' + '%d' % self._languages[l]['sloc'] + '\n' + + return desc + + + def process(self, msg): + """Collect code metrics.""" + + # Get all files + + all_files = [] + + baselen = len(self._rootdir) + len(os.sep) + + for root, dirnames, filenames in os.walk(self._rootdir): + + if '.git' in root: + continue + + for filename in filenames: + + full_access = os.path.join(root, filename) + rel_access = full_access[baselen:] + + all_files.append(rel_access) + + # Compute metrics + + count = len(all_files) + + i = 0 + + for filename in all_files: + + print(msg + ' -> Collecting code metrics... %d%%' % ((i * 100) / count), end='') + + unsupported = { + '.l' : 'Flex', + '.y' : 'Bison' + } + + supported = True + + lang = None + + for ext, lang in unsupported.items(): + + if filename.endswith(ext): + + supported = False + break + + if supported: + + stats = self._analyse_supported_file(filename) + + if stats is None: + i += 1 + continue + + lang = stats['language'] + + else: + + stats = self._analyse_unsupported_file(os.path.join(self._rootdir, filename)) + + if not(lang in self._languages): + + self._languages[lang] = {} + + self._languages[lang]['sloc'] = 0 + self._languages[lang]['comments'] = 0 + + self._languages[lang]['sloc'] += stats['sloc'] + self._languages[lang]['comments'] += stats['comments'] + + i += 1 + + msg = msg + ' -> Collecting code metrics... %d%%' % ((i * 100) / count) + + print(msg, end='') + + return msg + + + def _analyse_supported_file(self, filename): + """Process a file supported by the Python module.""" + + context = {} + + context['include_metrics'] = [ ('sloc', 'SLOCMetric') ] + + context['quiet'] = True + context['verbose'] = False + context['root_dir'] = self._rootdir + context['in_file_names'] = [ filename ] + context['output_format'] = None + + stats = process(context) + + if filename in stats: + + stats = stats[filename] + + if stats['language'] == 'TASM': + stats['language'] = 'Asm' + + if stats['language'] == 'Gettext Catalog': + stats = None + + else: + stats = None + + return stats + + + def _analyse_unsupported_file(self, filename): + """Process a file unsupported by the Python module.""" + + stats = {} + + stats['sloc'] = 0 + stats['comments'] = 0 + + with open(filename, 'r') as fin: + + for line in fin: + + length = len(line) + + if not(length == 0 or (length == 1 and line == '\n') or (length == 2 and line == '\r\n')): + stats['sloc'] += 1 + + return stats + + + def count_all_lines(self): + """Count all single lines of code.""" + + result = 0 + + for l in self._languages: + + result += self._languages[l]['sloc'] + + return result + + + def get_most_used(self, count): + """Compute the list of most used languages.""" + + max_count = len(self._languages.keys()) + + if count > max_count: + count = max_count + + languages = [] + + for l in self._languages: + + languages.append((self._languages[l]['sloc'], l)) + + languages = sorted(languages, reverse=True) + + if count < max_count: + + selected = languages[:count - 1] + + remaining = 0 + + for n, _ in languages[count:]: + + remaining += n + + selected.append((remaining, 'Others')) + + else: + + selected = languages + + return selected diff --git a/gas.py b/gas.py index 40509e8..ffb4ba6 100644 --- a/gas.py +++ b/gas.py @@ -1,77 +1,105 @@ # -*- coding: utf-8 -*- import argparse +import os +import pickle from board import DynamicFlyer - - - - +from repository import RepositoryChecker if __name__ == '__main__': parser = argparse.ArgumentParser() + parser.add_argument('--repo', help='Set the Git repository to process', type=str, required=True) + parser.add_argument('--year', help='Set the year to pay attention to', type=int, required=True) parser.add_argument('--header', help='Define the main title of the picture', type=str, required=True) parser.add_argument('--footer', help='Define a conclusion for the picture', type=str, required=True) parser.add_argument('--output', help='Set the output filename', type=str, required=True) args = parser.parse_args() + cache = args.repo + '.pickle' + if os.path.isfile(cache): - flyer = DynamicFlyer(250) + checker = pickle.load(open(cache, 'rb')) - flyer.print_header(args.header) + else: - data = { - 100 : 100, - 200 : 123, - 400 : 700, - 600 : 650, - 900 : 900 - } + checker = RepositoryChecker(args.repo, args.year) + checker.process() - color = (0.5, 0.2, 0.7) + pickle.dump(checker, open(cache, 'wb')) - color = (0.047, 0.365, 0.533) + # Colors - flyer.render_commit_timeline(0, 1000, data, False, color) + red = (1.0, 0.353, 0.165) + blue = (0.047, 0.365, 0.533) + green = (0.118, 0.71, 0.392) + grey = (0.6, 0.6, 0.6) - colors = [ (0.812, 0.176, 0.08), (0.04, 0.58, 0.255) ] + # Draw now! - colors = [ (0.643, 0.133, 0), (0, 0.459, 0.20) ] + flyer = DynamicFlyer(250) - colors = [ (1.0, 0.353, 0.165), (0.118, 0.71, 0.392) ] + flyer.print_header(args.header) - data = { - 'Insertions' : 794, - 'Deletions' : 312 - } + flyer.render_commit_timeline(checker.get('start'), checker.get('end'), checker.get('timeline'), False, blue) - flyer.render_pie('Code', data, colors) + # Code data = { - 'Added' : 100, - 'Removed' : 13 + 'Insertions' : (checker.get('insertions'), green), + 'Deletions' : (checker.get('deletions'), red) } - flyer.render_pie('Files', data, colors) + flyer.render_pie('Code', data) + + print('[i] Code:', checker.get('insertions'), checker.get('deletions')) + + # Files data = { - 'C' : 2489, - 'Python' : 631 + 'Added' : (checker.get('added'), green), + 'Removed' : (checker.get('removed'), red) } - flyer.render_pie('Languages', data, colors) + flyer.render_pie('Files', data) + + print('[i] Files:', checker.get('added'), checker.get('removed')) + + # Languages + + used = checker.get('last_metrics').get_most_used(4) + + data = {} + + colors = (green, red, blue, grey) + + i = 0 + + for v, n in used: + + data[n] = (v, colors[i]) + + i += 1 + + flyer.render_pie('Languages', data) + + # TODO / FIXME data = { - 'Remaining' : 89, - 'Killed' : 31 + 'Old' : (checker.get('old'), grey), + 'New' : (checker.get('new'), red), + 'Killed' : (checker.get('killed'), green) } - flyer.render_pie('TODO / FIXME', data, colors) + flyer.render_pie('TODO / FIXME', data) + + print('[i] TODO / FIXME:', checker.get('old'), checker.get('new'), checker.get('killed')) + # Conclusion flyer.print_footer(args.footer) diff --git a/repository.py b/repository.py new file mode 100644 index 0000000..67d7b88 --- /dev/null +++ b/repository.py @@ -0,0 +1,307 @@ +# -*- coding: utf-8 -*- + +import calendar +import difflib +import os +import shutil +import time +from code import CodeMetrics +from git import Repo + + +class RepositoryChecker: + """Browse a Git repository.""" + + + def __init__(self, url, year): + """Initialize the analysis of a Git repository.""" + + self._repo = Repo(url) + self._dump_path = url + '_dump' + + self._start = calendar.timegm(time.strptime('1/1/%u' % year, '%d/%m/%Y')) + self._end = calendar.timegm(time.strptime('1/1/%u' % (year + 1), '%d/%m/%Y')) + + self._timeline = {} + + self._last_metrics = None + + self._insertions = 0 + self._deletions = 0 + + self._added = 0 + self._removed = 0 + + self._old = 0 + self._new = 0 + self._killed = 0 + + + def __str__(self): + """Provide a pretty print of the checker.""" + + desc = '' + + desc += '=== Code ===\n' + desc += 'Insertions: %u\n' % self._insertions + desc += 'Deletions: %u\n' % self._deletions + + desc += '\n' + + desc += '=== Files ===\n' + desc += 'Added: %u\n' % self._added + desc += 'Removed: %u\n' % self._removed + + desc += '\n' + + desc += '=== TODO/FIXME ===\n' + desc += 'Old: %u\n' % self._old + desc += 'New: %u\n' % self._new + desc += 'Killed: %u\n' % self._killed + + return desc + + + def process(self): + """Collect Git info.""" + + selection = list(self._repo.iter_commits('master')) + + count = 0 + + previous = None + + for commit in list(selection): + + valid = self._start <= commit.committed_date and commit.committed_date < self._end + + if not(previous or valid): + continue + + elif previous and not(valid): + break + + previous = commit + + count += 1 + + i = 0 + + previous = None + + for commit in list(selection): + + valid = self._start <= commit.committed_date and commit.committed_date < self._end + + if not(previous or valid): + continue + + elif previous and not(valid): + break + + previous = commit + + print('\r[+] Collecting Git info... %d%%' % ((i * 100) / count), end='') + + # Insertions / deletions + + total = commit.stats.total + + self._insertions += total.get('insertions', 0) + + self._deletions += total.get('deletions', 0) + + # The run command is : + # + # git diff-tree SHA SHA~1 -r --abbrev=40 --full-index -M --raw --no-color + # + # Beware: all is reversed! + + for diff in commit.diff(commit.hexsha + '~1', create_patch=True, ignore_blank_lines=True, + ignore_space_at_eol=True, diff_filter='cr'): + + # Added / removed + + if diff.new_file: + self._removed += 1 + + if diff.deleted_file: + self._added += 1 + + # TODO / FIXME / REMME + + blob_a = None + blob_b = None + + try: + + if diff.a_blob: + blob_a = diff.a_blob.data_stream.read().decode('utf-8').splitlines(1) + + if diff.b_blob: + blob_b = diff.b_blob.data_stream.read().decode('utf-8').splitlines(1) + + except UnicodeDecodeError: + pass + + if blob_a is None and blob_b is None: + + # Binary file + pass + + elif blob_a is None: + + for line in blob_b: + + if 'TODO' in line or 'FIXME' in line or 'REMME' in line: + + self._killed += 1 + + elif blob_b is None: + + for line in blob_a: + + if 'TODO' in line or 'FIXME' in line or 'REMME' in line: + + self._new += 1 + + else: + + for line in difflib.unified_diff(blob_a, blob_b): + + if line.startswith('+++') or line.startswith('---'): + continue + + if 'TODO' in line or 'FIXME' in line or 'REMME' in line: + + if line.startswith('-'): + self._new += 1 + + elif line.startswith('+'): + self._killed += 1 + + # Single lines of code + + progress = '\r[+] Collecting Git info... %d%%' % ((i * 100) / count) + + self._delete_dump() + + msg = self._build_dump(commit.tree, progress) + + print('\r' + ' ' * len(msg[1:]), end='') + + print(progress, end='') + + cm = CodeMetrics(self._dump_path) + + msg = cm.process(progress) + + print('\r' + ' ' * len(msg[1:]), end='') + + print(progress, end='') + + self._timeline[commit.committed_date] = cm.count_all_lines() + + if self._last_metrics is None: + + self._last_metrics = cm + + # All remaining TODO / FIXME + + msg = self._grep_for_toto_fixme(commit.tree, progress) + + print('\r' + ' ' * len(msg[1:]), end='') + + print(progress, end='') + + i += 1 + + print('\r[+] Collecting Git info... %d%%' % ((i * 100) / count)) + + + def _delete_dump(self): + """Delete all dumped items.""" + + if os.path.exists(self._dump_path): + shutil.rmtree(self._dump_path) + + + def _build_dump(self, tree, msg): + """Dump all items from a commit tree.""" + + if not os.path.exists(self._dump_path): + os.makedirs(self._dump_path) + + count = len(list(tree.traverse())) + + i = 0 + + for item in tree.traverse(): + + print(msg + ' -> Dumping items... %d%%' % ((i * 100) / count), end='') + + path = os.path.join(self._dump_path, item.path) + + if item.type == 'tree': + + if not os.path.exists(path): + os.makedirs(path) + + elif item.type == 'blob': + + with open(path, 'wb') as out: + + out.write(item.data_stream.read()) + + i += 1 + + msg = msg + ' -> Dumping items... %d%%' % ((i * 100) / count) + + print(msg, end='') + + return msg + + + def _grep_for_toto_fixme(self, tree, msg): + """Find all waiting TODO / FIXME markers.""" + + count = len(list(tree.traverse())) + + i = 0 + + for item in tree.traverse(): + + print(msg + ' -> Searching for markers... %d%%' % ((i * 100) / count), end='') + + if item.type == 'blob': + + blob = None + + try: + + blob = item.data_stream.read().decode('utf-8').splitlines(1) + + except UnicodeDecodeError: + pass + + if blob: + + for line in blob: + + if 'TODO' in line or 'FIXME' in line or 'REMME' in line: + + self._old += 1 + + i += 1 + + msg = msg + ' -> Searching for markers... %d%%' % ((i * 100) / count) + + print(msg, end='') + + return msg + + + def get(self, name): + """Provide a memorized property.""" + + return getattr(self, '_' + name) -- cgit v0.11.2-87-g4458