diff options
| author | Cyrille Bagard <nocbos@gmail.com> | 2018-01-15 23:59:59 (GMT) | 
|---|---|---|
| committer | Cyrille Bagard <nocbos@gmail.com> | 2018-01-15 23:59:59 (GMT) | 
| commit | 169bab9bffb4b987e87448525e59813a658edb76 (patch) | |
| tree | 65838243872e375e618f424827397ec6a5c41f3b | |
| parent | 15b01266e1d7e193280658f300fd7d95bd918626 (diff) | |
| -rw-r--r-- | board.py | 159 | ||||
| -rwxr-xr-x | chrysalide.sh | 4 | ||||
| -rw-r--r-- | code.py | 206 | ||||
| -rw-r--r-- | gas.py | 94 | ||||
| -rw-r--r-- | repository.py | 307 | 
5 files changed, 675 insertions, 95 deletions
| @@ -99,8 +99,8 @@ class DynamicFlyer:          hmargin = self._width * 0.1          vmargin = self._square * 0.1 -        width = self._width - 2 * hmargin -        height = self._square - 2 * vmargin +        width = self._width - 2 * hmargin - 10 +        height = self._square - 2 * vmargin - 10          self._cr.save() @@ -108,37 +108,40 @@ class DynamicFlyer:          # Find the max and normalize +        min_vol = min(data.values())          max_vol = max(data.values()) +        diff_vol = max_vol - min_vol +          sx = width / (end - start) -        sy = height / max_vol +        sy = height / diff_vol + +        # Display volumes          self._cr.save()          self._cr.scale(sx, sy) -        # Display volumes -          first = min(data.keys())          if created: -            self._cr.move_to(first, max_vol) +            self._cr.move_to(first - start, diff_vol)          else: -            self._cr.move_to(0, max_vol) +            self._cr.move_to(0, diff_vol)              self._cr.line_to(0, max_vol - data[first])          for time in sorted(data.keys()):              vol = data[time] -            self._cr.line_to(time, max_vol - vol) +            self._cr.line_to(time - start, max_vol - vol)          last = max(data.keys()) -        self._cr.line_to(last, max_vol) +        self._cr.line_to(last - start, diff_vol)          self._cr.close_path() @@ -146,19 +149,9 @@ class DynamicFlyer:          self._cr.fill_preserve()          self._cr.set_source_rgba(*color, 1.0) -        self._cr.set_line_width(1 / sx) +        self._cr.set_line_width(1 / (sx if sx > sy else sy))          self._cr.stroke() -        # Display commits - -        for time, vol in data.items(): - -            self._cr.rectangle(time - 2 / sx, max_vol - vol - 2 / sy, 4 / sx, 4 / sy) - -            self._cr.set_source_rgba(*color, 1.0) -            self._cr.set_line_width(0.5) -            self._cr.fill() -          self._cr.restore()          # Display the borders @@ -170,26 +163,52 @@ class DynamicFlyer:          self._cr.set_source_rgba(*self._forecolor, 0.7)          self._cr.stroke() -        self._cr.move_to(0, 0) -        self._cr.line_to(-4, 10) -        self._cr.line_to(4, 10) +        self._cr.move_to(0, -10) +        self._cr.line_to(-4, 0) +        self._cr.line_to(4, 0)          self._cr.set_source_rgba(*self._forecolor, 1.0)          self._cr.fill() -        self._cr.move_to(width, height) -        self._cr.line_to(width - 10, height - 4) -        self._cr.line_to(width - 10, height + 4) +        self._cr.move_to(width + 10, height) +        self._cr.line_to(width, height - 4) +        self._cr.line_to(width, height + 4)          self._cr.set_source_rgba(*self._forecolor, 1.0)          self._cr.fill() +        # Display some time marks + +        unit = width / 12 + +        for i in range(11): + +            self._cr.move_to((i + 1) * unit, height - 2) +            self._cr.line_to((i + 1) * unit, height + 2) + +            self._cr.set_source_rgba(*self._forecolor, 1.0) +            self._cr.stroke() + +        # Display commits + +        self._cr.save() + +        self._cr.scale(sx, sy) + +        for time, vol in data.items(): + +            self._cr.rectangle(time - start - 2 / sx, max_vol - vol - 2 / sy, 4 / sx, 4 / sy) + +            self._cr.set_source_rgba(*color, 1.0) +            self._cr.fill() +          self._cr.restore()          # Print a caption as explaination +        self._cr.restore() +          count = len(data.values()) -        min_vol = min(data.values())          unit = '' @@ -232,7 +251,7 @@ class DynamicFlyer:          self._cr.restore() -    def _render_pie(self, title, data, colors): +    def _render_pie(self, title, data):          """Render a pie chart."""          cx = self._square / 2 @@ -249,10 +268,12 @@ class DynamicFlyer:          for k, v in data.items(): -            stats.append((v, k)) +            val, color = v + +            stats.append((val, k, color))              count += 1 -            total += v +            total += val          stats = sorted(stats) @@ -260,33 +281,38 @@ class DynamicFlyer:          done = 0 -        init_angle = - (stats[0][0] * math.pi) / (2 * total) +        init_angle = - (stats[0][0] * 2 * math.pi) / (total * 2)          last_angle = init_angle          for stat in stats: -            angle1 = last_angle +            value, _, color = stat -            if (done + 1) == count: -                angle2 = init_angle +            if value > 0: -            else: -                angle2 = angle1 + (stat[0] * math.pi) / total +                angle1 = last_angle -            self._cr.move_to(cx, cy) -            self._cr.arc(cx, cy, radius, angle1, angle2) -            self._cr.line_to(cx, cy) +                if (done + 1) == count: +                    angle2 = init_angle if init_angle != 0 else 2 * math.pi -            self._cr.set_source_rgba(*colors[done], 0.5) -            self._cr.fill_preserve() +                else: +                    angle2 = angle1 + (value * 2 * math.pi) / total -            self._cr.set_source_rgba(*self._backcolor, 1.0) -            self._cr.set_line_width(2) -            self._cr.stroke() +                self._cr.move_to(cx, cy) +                self._cr.arc(cx, cy, radius, angle1, angle2) +                self._cr.line_to(cx, cy) + +                self._cr.set_source_rgba(*color, 0.5) +                self._cr.fill_preserve() + +                self._cr.set_source_rgba(*self._backcolor, 1.0) +                self._cr.set_line_width(2) +                self._cr.stroke() + +                last_angle = angle2              done += 1 -            last_angle = angle2          done = 0 @@ -294,22 +320,27 @@ class DynamicFlyer:          for stat in stats: -            angle1 = last_angle +            value, _, color = stat -            if (done + 1) == count: -                angle2 = init_angle +            if value > 0: -            else: -                angle2 = angle1 + (stat[0] * math.pi) / total +                angle1 = last_angle -            self._cr.arc(cx, cy, radius, angle1, angle2) +                if (done + 1) == count: +                    angle2 = init_angle if init_angle != 0 else 2 * math.pi -            self._cr.set_source_rgba(*colors[done], 1.0) -            self._cr.set_line_width(10) -            self._cr.stroke() +                else: +                    angle2 = angle1 + (value * 2 * math.pi) / total + +                self._cr.arc(cx, cy, radius, angle1, angle2) + +                self._cr.set_source_rgba(*color, 1.0) +                self._cr.set_line_width(10) +                self._cr.stroke() + +                last_angle = angle2              done += 1 -            last_angle = angle2          # Printing the title @@ -346,7 +377,9 @@ class DynamicFlyer:          for stat in stats: -            x_off, y_off, tw, th = self._cr.text_extents(stat[1])[:4] +            _, name, _ = stat + +            x_off, y_off, tw, th = self._cr.text_extents(name)[:4]              if full_width > 0:                  full_width += self._caption_sep @@ -365,7 +398,9 @@ class DynamicFlyer:          for stat in stats: -            x_off, y_off, tw, th = self._cr.text_extents(stat[1])[:4] +            _, name, color = stat + +            x_off, y_off, tw, th = self._cr.text_extents(name)[:4]              self._cr.save() @@ -378,10 +413,10 @@ class DynamicFlyer:              self._cr.rectangle(0, top, self._caption_width, self._caption_width / 2) -            self._cr.set_source_rgba(*colors[done], 0.7) +            self._cr.set_source_rgba(*color, 0.7)              self._cr.fill_preserve() -            self._cr.set_source_rgba(*colors[done], 1.0) +            self._cr.set_source_rgba(*color, 1.0)              self._cr.set_line_width(1)              self._cr.stroke() @@ -391,7 +426,7 @@ class DynamicFlyer:              self._cr.set_source_rgba(*self._forecolor, 1.0) -            self._cr.show_text(stat[1]) +            self._cr.show_text(name)              self._cr.restore() @@ -400,12 +435,12 @@ class DynamicFlyer:              done += 1 -    def render_pie(self, title, data, colors): +    def render_pie(self, title, data):          """Render a pie chart."""          self._book_room(self._square, self._square) -        self._render_pie(title, data, colors) +        self._render_pie(title, data)          self._cr.restore() diff --git a/chrysalide.sh b/chrysalide.sh index b43716c..d821e4c 100755 --- a/chrysalide.sh +++ b/chrysalide.sh @@ -2,7 +2,11 @@  OUTPUT="2017-2018.png" +[ -d chrysalide ] || git clone http://git.0xdeadc0de.fr/chrysalide.git +  python3 ./gas.py \ +        --repo "$PWD/chrysalide" \ +        --year 2017 \          --header "Chrysalide's Git repository in 2017" \          --footer "Get more info at chrysalide.re" \          --output $OUTPUT @@ -0,0 +1,206 @@ +# -*- coding: utf-8 -*- + +import os +from metrics.metrics import process + + +class CodeMetrics: +    """Handle source code metrics.""" + + +    def __init__(self, rootdir): +        """Init the code metrics.""" + +        self._rootdir = rootdir + +        self._languages = {} + + +    def __str__(self): +        """Display the collected metrics.""" + +        desc = '' + +        for l in self._languages: + +            desc += l + ': ' + '%d' % self._languages[l]['sloc'] + '\n' + +        return desc + + +    def process(self, msg): +        """Collect code metrics.""" + +        # Get all files + +        all_files = [] + +        baselen = len(self._rootdir) + len(os.sep) + +        for root, dirnames, filenames in os.walk(self._rootdir): + +            if '.git' in root: +                continue + +            for filename in filenames: + +                full_access = os.path.join(root, filename) +                rel_access = full_access[baselen:] + +                all_files.append(rel_access) + +        # Compute metrics + +        count = len(all_files) + +        i = 0 + +        for filename in all_files: + +            print(msg + ' -> Collecting code metrics... %d%%' % ((i * 100) / count), end='') + +            unsupported = { +                '.l' : 'Flex', +                '.y' : 'Bison' +            } + +            supported = True + +            lang = None + +            for ext, lang in unsupported.items(): + +                if filename.endswith(ext): + +                    supported = False +                    break + +            if supported: + +                stats = self._analyse_supported_file(filename) + +                if stats is None: +                    i += 1 +                    continue + +                lang = stats['language'] + +            else: + +                stats = self._analyse_unsupported_file(os.path.join(self._rootdir, filename)) + +            if not(lang in self._languages): + +                self._languages[lang] = {} + +                self._languages[lang]['sloc'] = 0 +                self._languages[lang]['comments'] = 0 + +            self._languages[lang]['sloc'] += stats['sloc'] +            self._languages[lang]['comments'] += stats['comments'] + +            i += 1 + +        msg = msg + ' -> Collecting code metrics... %d%%' % ((i * 100) / count) + +        print(msg, end='') + +        return msg + + +    def _analyse_supported_file(self, filename): +        """Process a file supported by the Python module.""" + +        context = {} + +        context['include_metrics'] = [ ('sloc', 'SLOCMetric') ] + +        context['quiet'] = True +        context['verbose'] = False +        context['root_dir'] = self._rootdir +        context['in_file_names'] = [ filename ] +        context['output_format'] = None + +        stats = process(context) + +        if filename in stats: + +            stats = stats[filename] + +            if stats['language'] == 'TASM': +                stats['language'] = 'Asm' + +            if stats['language'] == 'Gettext Catalog': +                stats = None + +        else: +            stats = None + +        return stats + + +    def _analyse_unsupported_file(self, filename): +        """Process a file unsupported by the Python module.""" + +        stats = {} + +        stats['sloc'] = 0 +        stats['comments'] = 0 + +        with open(filename, 'r') as fin: + +            for line in fin: + +                length = len(line) + +                if not(length == 0 or (length == 1 and line == '\n') or (length == 2 and line == '\r\n')): +                    stats['sloc'] += 1 + +        return stats + + +    def count_all_lines(self): +        """Count all single lines of code.""" + +        result = 0 + +        for l in self._languages: + +            result += self._languages[l]['sloc'] + +        return result + + +    def get_most_used(self, count): +        """Compute the list of most used languages.""" + +        max_count = len(self._languages.keys()) + +        if count > max_count: +            count = max_count + +        languages = [] + +        for l in self._languages: + +            languages.append((self._languages[l]['sloc'], l)) + +        languages = sorted(languages, reverse=True) + +        if count < max_count: + +            selected = languages[:count - 1] + +            remaining = 0 + +            for n, _ in languages[count:]: + +                remaining += n + +            selected.append((remaining, 'Others')) + +        else: + +            selected = languages + +        return selected @@ -1,77 +1,105 @@  # -*- coding: utf-8 -*-  import argparse +import os +import pickle  from board import DynamicFlyer - - - - +from repository import RepositoryChecker  if __name__ == '__main__':      parser = argparse.ArgumentParser() +    parser.add_argument('--repo',   help='Set the Git repository to process', type=str, required=True) +    parser.add_argument('--year',   help='Set the year to pay attention to', type=int, required=True)      parser.add_argument('--header', help='Define the main title of the picture', type=str, required=True)      parser.add_argument('--footer', help='Define a conclusion for the picture', type=str, required=True)      parser.add_argument('--output', help='Set the output filename', type=str, required=True)      args = parser.parse_args() +    cache = args.repo + '.pickle' +    if os.path.isfile(cache): -    flyer = DynamicFlyer(250) +        checker = pickle.load(open(cache, 'rb')) -    flyer.print_header(args.header) +    else: -    data = { -        100 : 100, -        200 : 123, -        400 : 700, -        600 : 650, -        900 : 900 -    } +        checker = RepositoryChecker(args.repo, args.year) +        checker.process() -    color = (0.5, 0.2, 0.7) +        pickle.dump(checker, open(cache, 'wb')) -    color = (0.047, 0.365, 0.533) +    # Colors -    flyer.render_commit_timeline(0, 1000, data, False, color) +    red = (1.0, 0.353, 0.165) +    blue = (0.047, 0.365, 0.533) +    green = (0.118, 0.71, 0.392) +    grey = (0.6, 0.6, 0.6) -    colors = [ (0.812, 0.176, 0.08), (0.04, 0.58, 0.255) ] +    # Draw now! -    colors = [ (0.643, 0.133, 0), (0, 0.459, 0.20) ] +    flyer = DynamicFlyer(250) -    colors = [ (1.0, 0.353, 0.165), (0.118, 0.71, 0.392) ] +    flyer.print_header(args.header) -    data = { -        'Insertions' : 794, -        'Deletions' : 312 -    } +    flyer.render_commit_timeline(checker.get('start'), checker.get('end'), checker.get('timeline'), False, blue) -    flyer.render_pie('Code', data, colors) +    # Code      data = { -        'Added' : 100, -        'Removed' : 13 +        'Insertions' : (checker.get('insertions'), green), +        'Deletions' :  (checker.get('deletions'), red)      } -    flyer.render_pie('Files', data, colors) +    flyer.render_pie('Code', data) + +    print('[i] Code:', checker.get('insertions'), checker.get('deletions')) + +    # Files      data = { -        'C' : 2489, -        'Python' : 631 +        'Added' :   (checker.get('added'), green), +        'Removed' : (checker.get('removed'), red)      } -    flyer.render_pie('Languages', data, colors) +    flyer.render_pie('Files', data) + +    print('[i] Files:', checker.get('added'), checker.get('removed')) + +    # Languages + +    used = checker.get('last_metrics').get_most_used(4) + +    data = {} + +    colors = (green, red, blue, grey) + +    i = 0 + +    for v, n in used: + +        data[n] = (v, colors[i]) + +        i += 1 + +    flyer.render_pie('Languages', data) + +    # TODO / FIXME      data = { -        'Remaining' : 89, -        'Killed' : 31 +        'Old' :    (checker.get('old'), grey), +        'New' :    (checker.get('new'), red), +        'Killed' : (checker.get('killed'), green)      } -    flyer.render_pie('TODO / FIXME', data, colors) +    flyer.render_pie('TODO / FIXME', data) + +    print('[i] TODO / FIXME:', checker.get('old'), checker.get('new'), checker.get('killed')) +    # Conclusion      flyer.print_footer(args.footer) diff --git a/repository.py b/repository.py new file mode 100644 index 0000000..67d7b88 --- /dev/null +++ b/repository.py @@ -0,0 +1,307 @@ +# -*- coding: utf-8 -*- + +import calendar +import difflib +import os +import shutil +import time +from code import CodeMetrics +from git import Repo + + +class RepositoryChecker: +    """Browse a Git repository.""" + + +    def __init__(self, url, year): +        """Initialize the analysis of a Git repository.""" + +        self._repo = Repo(url) +        self._dump_path = url + '_dump' + +        self._start = calendar.timegm(time.strptime('1/1/%u' % year, '%d/%m/%Y')) +        self._end = calendar.timegm(time.strptime('1/1/%u' % (year + 1), '%d/%m/%Y')) + +        self._timeline = {} + +        self._last_metrics = None + +        self._insertions = 0 +        self._deletions = 0 + +        self._added = 0 +        self._removed = 0 + +        self._old = 0 +        self._new = 0 +        self._killed = 0 + + +    def __str__(self): +        """Provide a pretty print of the checker.""" + +        desc = '' + +        desc += '=== Code ===\n' +        desc += 'Insertions: %u\n' % self._insertions +        desc += 'Deletions:  %u\n' % self._deletions + +        desc += '\n' + +        desc += '=== Files ===\n' +        desc += 'Added:    %u\n' % self._added +        desc += 'Removed:  %u\n' % self._removed + +        desc += '\n' + +        desc += '=== TODO/FIXME ===\n' +        desc += 'Old:    %u\n' % self._old +        desc += 'New:    %u\n' % self._new +        desc += 'Killed: %u\n' % self._killed + +        return desc + + +    def process(self): +        """Collect Git info.""" + +        selection = list(self._repo.iter_commits('master')) + +        count = 0 + +        previous = None + +        for commit in list(selection): + +            valid = self._start <= commit.committed_date and commit.committed_date < self._end + +            if not(previous or valid): +                continue + +            elif previous and not(valid): +                break + +            previous = commit + +            count += 1 + +        i = 0 + +        previous = None + +        for commit in list(selection): + +            valid = self._start <= commit.committed_date and commit.committed_date < self._end + +            if not(previous or valid): +                continue + +            elif previous and not(valid): +                break + +            previous = commit + +            print('\r[+] Collecting Git info... %d%%' % ((i * 100) / count), end='') + +            # Insertions / deletions + +            total = commit.stats.total + +            self._insertions += total.get('insertions', 0) + +            self._deletions += total.get('deletions', 0) + +            # The run command is : +            # +            #   git diff-tree SHA SHA~1 -r --abbrev=40 --full-index -M --raw --no-color +            # +            # Beware: all is reversed! + +            for diff in commit.diff(commit.hexsha + '~1', create_patch=True, ignore_blank_lines=True,  +                                    ignore_space_at_eol=True, diff_filter='cr'): + +                # Added / removed + +                if diff.new_file: +                    self._removed += 1 + +                if diff.deleted_file: +                    self._added += 1 + +                # TODO / FIXME / REMME + +                blob_a = None +                blob_b = None + +                try: + +                    if diff.a_blob: +                        blob_a = diff.a_blob.data_stream.read().decode('utf-8').splitlines(1) + +                    if diff.b_blob: +                        blob_b = diff.b_blob.data_stream.read().decode('utf-8').splitlines(1) + +                except UnicodeDecodeError: +                    pass + +                if blob_a is None and blob_b is None: + +                    # Binary file +                    pass + +                elif blob_a is None: + +                    for line in blob_b: + +                        if 'TODO' in line or 'FIXME' in line or 'REMME' in line: + +                            self._killed += 1 + +                elif blob_b is None: + +                    for line in blob_a: + +                        if 'TODO' in line or 'FIXME' in line or 'REMME' in line: + +                            self._new += 1 + +                else: + +                    for line in difflib.unified_diff(blob_a, blob_b): + +                        if line.startswith('+++') or line.startswith('---'): +                            continue + +                        if 'TODO' in line or 'FIXME' in line or 'REMME' in line: + +                            if line.startswith('-'): +                                self._new += 1 + +                            elif line.startswith('+'): +                                self._killed += 1 + +            # Single lines of code + +            progress = '\r[+] Collecting Git info... %d%%' % ((i * 100) / count) + +            self._delete_dump() + +            msg = self._build_dump(commit.tree, progress) + +            print('\r' + ' ' * len(msg[1:]), end='') + +            print(progress, end='') + +            cm = CodeMetrics(self._dump_path) + +            msg = cm.process(progress) + +            print('\r' + ' ' * len(msg[1:]), end='') + +            print(progress, end='') + +            self._timeline[commit.committed_date] = cm.count_all_lines() + +            if self._last_metrics is None: + +                self._last_metrics = cm + +                # All remaining TODO / FIXME + +                msg = self._grep_for_toto_fixme(commit.tree, progress) + +                print('\r' + ' ' * len(msg[1:]), end='') + +                print(progress, end='') + +            i += 1 + +        print('\r[+] Collecting Git info... %d%%' % ((i * 100) / count)) + + +    def _delete_dump(self): +        """Delete all dumped items.""" + +        if os.path.exists(self._dump_path): +            shutil.rmtree(self._dump_path) + + +    def _build_dump(self, tree, msg): +        """Dump all items from a commit tree.""" + +        if not os.path.exists(self._dump_path): +            os.makedirs(self._dump_path) + +        count = len(list(tree.traverse())) + +        i = 0 + +        for item in tree.traverse(): + +            print(msg + ' -> Dumping items... %d%%' % ((i * 100) / count), end='') + +            path = os.path.join(self._dump_path, item.path) + +            if item.type == 'tree': + +                if not os.path.exists(path): +                    os.makedirs(path) + +            elif item.type == 'blob': + +                with open(path, 'wb') as out: + +                    out.write(item.data_stream.read()) + +            i += 1 + +        msg = msg + ' -> Dumping items... %d%%' % ((i * 100) / count) + +        print(msg, end='') + +        return msg + + +    def _grep_for_toto_fixme(self, tree, msg): +        """Find all waiting TODO / FIXME markers.""" + +        count = len(list(tree.traverse())) + +        i = 0 + +        for item in tree.traverse(): + +            print(msg + ' -> Searching for markers... %d%%' % ((i * 100) / count), end='') + +            if item.type == 'blob': + +                blob = None + +                try: + +                    blob = item.data_stream.read().decode('utf-8').splitlines(1) + +                except UnicodeDecodeError: +                    pass + +                if blob: + +                    for line in blob: + +                        if 'TODO' in line or 'FIXME' in line or 'REMME' in line: + +                            self._old += 1 + +            i += 1 + +        msg = msg + ' -> Searching for markers... %d%%' % ((i * 100) / count) + +        print(msg, end='') + +        return msg + + +    def get(self, name): +        """Provide a memorized property.""" + +        return getattr(self, '_' + name) | 
