diff options
| -rw-r--r-- | scripts/repo-issue-stats/INSTALL_SYSTEMD.md | 28 | ||||
| -rw-r--r-- | scripts/repo-issue-stats/README.md | 9 | ||||
| -rw-r--r-- | scripts/repo-issue-stats/update-issues.service | 20 | ||||
| -rw-r--r-- | scripts/repo-issue-stats/update-issues.timer | 14 | ||||
| -rwxr-xr-x | scripts/repo-issue-stats/update_issues.py | 147 | ||||
| -rwxr-xr-x | scripts/repo-issue-stats/visualize_issues_timeline.py | 383 |
6 files changed, 601 insertions, 0 deletions
diff --git a/scripts/repo-issue-stats/INSTALL_SYSTEMD.md b/scripts/repo-issue-stats/INSTALL_SYSTEMD.md new file mode 100644 index 0000000000..95521effb4 --- /dev/null +++ b/scripts/repo-issue-stats/INSTALL_SYSTEMD.md @@ -0,0 +1,28 @@ +# Installing systemd timer for updating the static html file + +## Installation steps + +1. **Copy the service and timer files to systemd directory:** + ```bash + sudo cp update-issues.service update-issues.timer /etc/systemd/system/ + ``` + +2. **Reload systemd to recognize new files:** + ```bash + sudo systemctl daemon-reload + ``` + +3. **Enable and start the timer:** + ```bash + sudo systemctl enable update-issues.timer + sudo systemctl start update-issues.timer + ``` + +## Uninstall + +```bash +sudo systemctl stop update-issues.timer +sudo systemctl disable update-issues.timer +sudo rm /etc/systemd/system/update-issues.service /etc/systemd/system/update-issues.timer +sudo rm sudo systemctl daemon-reload +``` diff --git a/scripts/repo-issue-stats/README.md b/scripts/repo-issue-stats/README.md new file mode 100644 index 0000000000..e24484db93 --- /dev/null +++ b/scripts/repo-issue-stats/README.md @@ -0,0 +1,9 @@ +# Github repository issue stats visualization + +These scripts create a static html page where you can view a graph over the number of open github issues on this repository. This static html file is regenerated nightly. + +When running `update_issues.py` the first time to bootstrap `mullvadvpn-app.issues/` you need to run it with a github token to be able to fetch all issues without getting rate limited. Subsequent updates (done by the systemd timer) can run without a token, since there usually are not that many new issues. + +``` +GITHUB_TOKEN=$(gh auth token) ./update_issues.py mullvad/mullvadvpn-app mullvadvpn-app.issues/ +``` diff --git a/scripts/repo-issue-stats/update-issues.service b/scripts/repo-issue-stats/update-issues.service new file mode 100644 index 0000000000..aaa187479e --- /dev/null +++ b/scripts/repo-issue-stats/update-issues.service @@ -0,0 +1,20 @@ +[Unit] +Description=Update GitHub issues for mullvadvpn-app and generate visualization +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/usr/local/mullvadvpn-app.issues/ + +# Environment variables +Environment="OUTPUT_HTML_PATH=/var/www/html/issues_timeline.html" + +# Step 1: Update issues from GitHub API +ExecStart=/usr/bin/python3 update_issues.py mullvad/mullvadvpn-app mullvadvpn-app.issues/ + +# Step 2: Generate HTML visualization +ExecStart=/usr/bin/python3 visualize_issues_timeline.py mullvadvpn-app.issues/ ${OUTPUT_HTML_PATH} + +StandardOutput=journal +StandardError=journal diff --git a/scripts/repo-issue-stats/update-issues.timer b/scripts/repo-issue-stats/update-issues.timer new file mode 100644 index 0000000000..d3749d62fb --- /dev/null +++ b/scripts/repo-issue-stats/update-issues.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Run GitHub issues update nightly +Requires=update-issues.service + +[Timer] +# Run at 2 AM every night +OnCalendar=daily +OnCalendar=*-*-* 02:00:00 + +# Run 5 minutes after boot if the system was off during scheduled time +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/repo-issue-stats/update_issues.py b/scripts/repo-issue-stats/update_issues.py new file mode 100755 index 0000000000..7b8b28394c --- /dev/null +++ b/scripts/repo-issue-stats/update_issues.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +""" +Update local GitHub issues by fetching new issues from the API. +Only downloads issues that don't already exist locally. +""" + +import requests +import json +import os +import sys +from pathlib import Path + + +def get_existing_issue_numbers(directory): + """Get a set of all issue numbers already saved locally.""" + issue_numbers = set() + + if not os.path.exists(directory): + return issue_numbers + + for filename in os.listdir(directory): + if filename.endswith('.json'): + try: + issue_num = int(filename[:-5]) # Remove .json extension + issue_numbers.add(issue_num) + except ValueError: + pass # Skip files that don't match the pattern + + return issue_numbers + + +def fetch_new_issues(repo, issues_dir, token=None): + """Fetch new issues from GitHub API and save them locally.""" + + # Get existing issue numbers + existing_issues = get_existing_issue_numbers(issues_dir) + + if existing_issues: + max_existing = max(existing_issues) + print(f"Found {len(existing_issues)} existing issues (highest: #{max_existing})") + else: + max_existing = 0 + print("No existing issues found") + + # Prepare API request + api_url = f"https://api.github.com/repos/{repo}/issues" + headers = {} + if token: + headers['Authorization'] = f'token {token}' + print("Using GitHub token for authentication") + else: + print("No GitHub token provided (rate limits apply)") + + # Create directory if it doesn't exist + os.makedirs(issues_dir, exist_ok=True) + + new_issues_count = 0 + page = 1 + + while True: + params = { + 'state': 'all', + 'per_page': 100, + 'sort': 'created', + 'direction': 'desc', + 'page': page + } + + print(f"Fetching page {page}...") + response = requests.get(api_url, params=params, headers=headers) + response.raise_for_status() + + # Check rate limit + remaining = response.headers.get('X-RateLimit-Remaining') + if remaining: + print(f" Rate limit remaining: {remaining}") + + issues = response.json() + + if not issues: + print("No more issues to fetch") + break + + # Process each issue + found_existing = False + for issue in issues: + issue_number = issue['number'] + + # Check if we already have this issue + if issue_number in existing_issues: + print(f" Found existing issue #{issue_number}, stopping fetch") + found_existing = True + break + + # Save new issue + filename = os.path.join(issues_dir, f"{issue_number}.json") + with open(filename, 'w') as f: + json.dump(issue, f, indent=4) + + new_issues_count += 1 + print(f" Saved issue #{issue_number}") + + # Stop if we found an existing issue + if found_existing: + break + + # Check if there are more pages + link_header = response.headers.get('Link', '') + if 'rel="next"' not in link_header: + print("Reached last page") + break + + page += 1 + + return new_issues_count + + +def main(): + if len(sys.argv) != 3: + print("Usage: update_issues.py <owner/repo> <issues_directory>") + print("Example: update_issues.py mullvad/mullvadvpn-app mullvadvpn-app.issues/") + print() + print("Optional: Set GITHUB_TOKEN environment variable for higher rate limits") + print(" export GITHUB_TOKEN=your_token_here") + sys.exit(1) + + repo = sys.argv[1] + issues_dir = sys.argv[2] + token = os.environ.get('GITHUB_TOKEN') + + try: + print(f"Updating issues for {repo}") + print(f"Target directory: {issues_dir}") + print() + + new_count = fetch_new_issues(repo, issues_dir, token) + + print() + print(f"Done! Downloaded {new_count} new issues") + + except Exception as e: + print(f"Error: {e}") + raise + + +if __name__ == "__main__": + main() diff --git a/scripts/repo-issue-stats/visualize_issues_timeline.py b/scripts/repo-issue-stats/visualize_issues_timeline.py new file mode 100755 index 0000000000..682e9fe8e9 --- /dev/null +++ b/scripts/repo-issue-stats/visualize_issues_timeline.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +""" +Visualize the number of open issues over time from local JSON files. +Generates an interactive HTML page with label filtering. +""" + +import json +import os +import sys +from datetime import datetime +from collections import defaultdict +from pathlib import Path + + +def load_issues_from_directory(directory): + """Load all issues from JSON files in the directory.""" + issues = [] + json_files = list(Path(directory).glob("*.json")) + + print(f"Loading issues from {directory}...") + for json_file in json_files: + try: + with open(json_file, 'r') as f: + issue = json.load(f) + # Filter out pull requests + if 'pull_request' not in issue: + issues.append(issue) + except Exception as e: + print(f" Warning: Failed to load {json_file}: {e}") + + print(f"Loaded {len(issues)} issues (excluding pull requests)") + return issues + + +def extract_labels(issues): + """Extract all unique labels from issues.""" + labels = set() + for issue in issues: + for label in issue.get('labels', []): + labels.add(label['name']) + + return sorted(labels) + + +def calculate_timeline_for_filter(issues, label_filter=None): + """Calculate the number of open issues over time, optionally filtered by label.""" + # Filter issues by label if specified + if label_filter: + filtered_issues = [ + issue for issue in issues + if any(label['name'] == label_filter for label in issue.get('labels', [])) + ] + else: + filtered_issues = issues + + events = [] + + # Create events for issue creation and closure + for issue in filtered_issues: + created_at = datetime.fromisoformat(issue['created_at'].replace('Z', '+00:00')) + events.append((created_at, 1)) # +1 for opening + + if issue['closed_at']: + closed_at = datetime.fromisoformat(issue['closed_at'].replace('Z', '+00:00')) + events.append((closed_at, -1)) # -1 for closing + + # Sort events by time + events.sort(key=lambda x: x[0]) + + # Calculate running total + timeline = [] + open_count = 0 + + for timestamp, delta in events: + open_count += delta + timeline.append({ + 'date': timestamp.strftime('%Y-%m-%d %H:%M:%S'), + 'count': open_count + }) + + return timeline + + +def generate_html(issues, labels, repo_name="mullvad/mullvadvpn-app"): + """Generate an HTML page with interactive label filtering.""" + + # Calculate timelines for all labels + print("Calculating timelines...") + timelines = {} + + # All issues + print(" All issues...") + timelines['All issues'] = calculate_timeline_for_filter(issues, None) + + # Each label + for label in labels: + print(f" {label}...") + timelines[label] = calculate_timeline_for_filter(issues, label) + + # Create fixed color mapping for each label + colors = [ + '#2196F3', '#4CAF50', '#FF9800', '#E91E63', '#9C27B0', + '#00BCD4', '#FF5722', '#795548', '#607D8B', '#FFC107', + '#8BC34A', '#3F51B5', '#F44336', '#009688', '#CDDC39', + '#FF6F00', '#673AB7', '#00897B', '#C62828', '#5E35B1', + '#D81B60', '#00ACC1', '#6D4C41', '#1565C0', '#EF6C00' + ] + color_map = {'All issues': colors[0]} + for i, label in enumerate(labels): + color_map[label] = colors[(i + 1) % len(colors)] + + html = f"""<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>Open Issues Over Time - {repo_name}</title> + <script src="https://cdn.plot.ly/plotly-2.27.0.min.js"></script> + <style> + body {{ + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Arial, sans-serif; + margin: 0; + padding: 20px; + background-color: #f5f5f5; + }} + .container {{ + max-width: 1200px; + margin: 0 auto; + background-color: white; + padding: 30px; + border-radius: 8px; + box-shadow: 0 2px 4px rgba(0,0,0,0.1); + }} + h1 {{ + color: #333; + margin-bottom: 10px; + }} + .subtitle {{ + color: #666; + margin-bottom: 20px; + }} + .controls {{ + margin-bottom: 20px; + padding: 20px; + background-color: #f9f9f9; + border-radius: 8px; + }} + .controls h3 {{ + margin-top: 0; + margin-bottom: 15px; + color: #333; + font-size: 16px; + }} + .checkbox-grid {{ + display: grid; + grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); + gap: 12px; + }} + .checkbox-item {{ + display: flex; + align-items: center; + }} + .checkbox-item input[type="checkbox"] {{ + width: 18px; + height: 18px; + margin-right: 8px; + cursor: pointer; + }} + .checkbox-item label {{ + cursor: pointer; + font-weight: normal; + color: #999; + user-select: none; + transition: color 0.2s; + }} + .checkbox-item input[type="checkbox"]:checked + label {{ + font-weight: 600; + }} + .checkbox-item.all-issues {{ + grid-column: 1 / -1; + padding-bottom: 10px; + border-bottom: 1px solid #ddd; + margin-bottom: 5px; + }} + .checkbox-item.all-issues label {{ + font-weight: 600; + }} + #chart {{ + width: 100%; + height: 600px; + }} + .stats {{ + margin-top: 20px; + padding: 15px; + background-color: #f9f9f9; + border-radius: 4px; + color: #666; + }} + </style> +</head> +<body> + <div class="container"> + <h1>Open Issues Over Time</h1> + <div class="subtitle">Repository: <a href="https://github.com/{repo_name}">{repo_name}</a></div> + + <div class="controls"> + <h3>Filter by labels:</h3> + <div class="checkbox-grid"> + <div class="checkbox-item all-issues"> + <input type="checkbox" id="filter-all-issues" value="All issues" checked data-color="{color_map['All issues']}"> + <label for="filter-all-issues">All issues</label> + </div> + {''.join(f'''<div class="checkbox-item"> + <input type="checkbox" id="filter-{label.replace(" ", "-").lower()}" value="{label}" data-color="{color_map[label]}"> + <label for="filter-{label.replace(" ", "-").lower()}">{label}</label> + </div>''' for label in labels)} + </div> + </div> + + <div id="chart"></div> + <div class="stats" id="stats"></div> + </div> + + <script> + const timelines = {json.dumps(timelines)}; + const colorMap = {json.dumps(color_map)}; + + function updateChart() {{ + // Get all checked checkboxes + const checkboxes = document.querySelectorAll('.checkbox-grid input[type="checkbox"]:checked'); + const selectedFilters = Array.from(checkboxes).map(cb => cb.value); + + if (selectedFilters.length === 0) {{ + document.getElementById('stats').innerHTML = 'Please select at least one filter.'; + Plotly.newPlot('chart', [], {{}}); + return; + }} + + // Create traces for each selected filter + const traces = selectedFilters.map(filter => {{ + const data = timelines[filter]; + + if (!data || data.length === 0) {{ + return null; + }} + + const dates = data.map(d => d.date); + const counts = data.map(d => d.count); + + return {{ + x: dates, + y: counts, + type: 'scatter', + mode: 'lines', + name: filter, + line: {{ + color: colorMap[filter], + width: 2 + }} + }}; + }}).filter(trace => trace !== null); + + const layout = {{ + title: '', + xaxis: {{ + title: 'Date', + showgrid: true, + gridcolor: '#e0e0e0' + }}, + yaxis: {{ + title: 'Number of Open Issues', + showgrid: true, + gridcolor: '#e0e0e0' + }}, + hovermode: 'closest', + legend: {{ + x: 0, + xanchor: 'left', + y: 1 + }}, + margin: {{ + l: 60, + r: 30, + t: 30, + b: 60 + }} + }}; + + const config = {{ + responsive: true, + displayModeBar: true, + displaylogo: false + }}; + + Plotly.newPlot('chart', traces, layout, config); + + // Update stats + let statsHtml = ''; + selectedFilters.forEach(filter => {{ + const data = timelines[filter]; + if (data && data.length > 0) {{ + const counts = data.map(d => d.count); + const currentCount = counts[counts.length - 1]; + statsHtml += `<strong>${{filter}}:</strong> ${{currentCount}} open | `; + }} + }}); + + document.getElementById('stats').innerHTML = statsHtml.slice(0, -3); // Remove trailing ' | ' + }} + + // Function to update label colors based on checkbox state + function updateLabelColors() {{ + document.querySelectorAll('.checkbox-grid input[type="checkbox"]').forEach(checkbox => {{ + const label = checkbox.nextElementSibling; + if (checkbox.checked) {{ + label.style.color = checkbox.getAttribute('data-color'); + }} else {{ + label.style.color = '#999'; + }} + }}); + }} + + // Initial setup + updateLabelColors(); + updateChart(); + + // Listen for checkbox changes + document.querySelectorAll('.checkbox-grid input[type="checkbox"]').forEach(checkbox => {{ + checkbox.addEventListener('change', function() {{ + updateLabelColors(); + updateChart(); + }}); + }}); + </script> +</body> +</html>""" + + return html + + +def main(): + if len(sys.argv) < 2: + print("Usage: visualize_issues_timeline.py <issues_directory> [output_file]") + print("Example: visualize_issues_timeline.py mullvadvpn-app.issues/") + sys.exit(1) + + issues_dir = sys.argv[1] + output_file = sys.argv[2] if len(sys.argv) > 2 else "issues_timeline.html" + + if not os.path.isdir(issues_dir): + print(f"Error: {issues_dir} is not a directory") + sys.exit(1) + + try: + # Load issues + issues = load_issues_from_directory(issues_dir) + + if not issues: + print("Error: No issues found") + sys.exit(1) + + # Extract labels + print("Extracting labels...") + labels = extract_labels(issues) + print(f"Found {len(labels)} unique labels: {', '.join(labels)}") + + # Generate HTML + print("Generating HTML...") + html = generate_html(issues, labels) + + # Write to file + with open(output_file, 'w') as f: + f.write(html) + + print(f"\nSuccess! Open {output_file} in your browser to view the visualization.") + + except Exception as e: + print(f"Error: {e}") + raise + + +if __name__ == "__main__": + main() |