diff --git a/README.md b/README.md index bbabfc7..6b199ee 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,22 @@ This repo helps us calculate point values from contributions from our various communication channels. +## Requirements + +- **Python 3** (all scripts require Python 3) +- Install dependencies: `pip install -r requirements.txt` + + | Script | Dependencies (beyond stdlib) | + |--------|------------------------------| + | github-stats.py | requests, python-dateutil | + | smartsheets-stats.py | smartsheet-python-sdk, python-dateutil | + | smartsheets-reports-stats.py | requests, python-dateutil | + | hangouts-chat.py | requests, oauth2client | + | trello-stats.py | requests, python-dateutil | + | rocketchat.py | requests, python-dateutil | + | gitlab-stats.py | requests, python-dateutil, pytz | + | mailman-subscribers.py | none (stdlib only) | + ## GitHub contributions For contributions to GitHub, we use search filters to find people's contributions. You can run these from [github.com/pulls](https://github.com/pulls) diff --git a/github-stats.py b/github-stats.py index fd504e6..b78a226 100755 --- a/github-stats.py +++ b/github-stats.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import os, json, requests, sys, argparse, re +import os, json, requests, sys, argparse, re, urllib.parse from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta @@ -12,13 +12,21 @@ DEFAULT_START_DATE_DAY = '01' UNLABELED = 'unlabeled' -def handle_pagination_items(session, url): -# print "pagination called: {}".format(url) - pagination_request = session.get(url) +def handle_pagination_items(session, url, params=None): +# print("pagination called: {}".format(url)) + pagination_request = session.get(url, params=params) + if pagination_request.status_code != 200: + # Print error details for debugging + try: + error_json = pagination_request.json() + print("Error: HTTP {} - {}".format(pagination_request.status_code, json.dumps(error_json, indent=2))) + except: + print("Error: HTTP {} - {}".format(pagination_request.status_code, pagination_request.text[:500])) pagination_request.raise_for_status() if 'next' in pagination_request.links and pagination_request.links['next']: - return pagination_request.json()['items'] + handle_pagination_items(session, pagination_request.links['next']['url']) + # Pagination links already contain the full URL with query params, so pass None + return pagination_request.json()['items'] + handle_pagination_items(session, pagination_request.links['next']['url'], params=None) else: return pagination_request.json()['items'] @@ -40,17 +48,17 @@ def valid_date(s): def encode_text(text): if text: - return text.encode("utf-8") - + # In Python 3, strings are already unicode, so just return as-is + return text return text def get_org_repos(session, github_org): - return handle_pagination_items(session, "https://api.github.com/orgs/{0}/repos".format(github_org)) + return handle_pagination_items(session, "https://api.github.com/orgs/{0}/repos".format(github_org), params=None) def get_org_members(session, github_org): - return handle_pagination_items(session, "https://api.github.com/orgs/{0}/members".format(github_org)) + return handle_pagination_items(session, "https://api.github.com/orgs/{0}/members".format(github_org), params=None) def get_pr(session, url): pr_request = session.get(url) @@ -64,10 +72,38 @@ def get_reviews(session, url): return pr_request.json() -def get_org_search_issues(session, start_date, github_org): +def build_search_query(github_org, date_str, issue_type): + """Build a GitHub search query string for a specific type (issue or pr)""" + query_parts = [ + "user:{}".format(github_org), + "updated:>={}".format(date_str), + "state:closed", + issue_type + ] + query_with_plus = "+".join(query_parts) + # URL-encode special characters but preserve + signs + encoded = urllib.parse.quote(query_with_plus, safe='') + # Replace %2B (encoded +) back to + since GitHub search API expects + for spaces + encoded_query = encoded.replace('%2B', '+') + return encoded_query - query = "https://api.github.com/search/issues?q=user:{}+updated:>={}+archived:false+state:closed&per_page=200".format(github_org, start_date.date().isoformat()) - return handle_pagination_items(session, query) +def get_org_search_issues(session, start_date, github_org): + # GitHub API now requires 'is:issue' or 'is:pull-request' in the query + # We need to search for issues and PRs separately, then combine results + date_str = start_date.date().isoformat() + + # Search for issues + issues_query = build_search_query(github_org, date_str, "is:issue") + issues_url = "https://api.github.com/search/issues?q={}&per_page=200".format(issues_query) + issues_results = handle_pagination_items(session, issues_url, params=None) + + # Search for pull requests + prs_query = build_search_query(github_org, date_str, "is:pr") + prs_url = "https://api.github.com/search/issues?q={}&per_page=200".format(prs_query) + prs_results = handle_pagination_items(session, prs_url, params=None) + + # Combine results + return issues_results + prs_results def process_labels(labels): label_dict = {} @@ -118,7 +154,7 @@ def repo_is_included(issue, repo_matcher, repo_excluder): repo_name = issue['repository_url'].split('/')[-1] repo_name_matches = True if re.match(repo_matcher, repo_name) != None else False repo_name_excluded = True if None != repo_excluder and re.match(repo_excluder, repo_name) != None else False - #print "{0} - matches? {1}, excluded? {2}".format(repo_name, repo_name_matches, repo_name_excluded) + #print("{0} - matches? {1}, excluded? {2}".format(repo_name, repo_name_matches, repo_name_excluded)) if repo_name_matches and repo_name_excluded == False: return True return False; @@ -149,7 +185,7 @@ def repo_is_included(issue, repo_matcher, repo_excluder): github_api_token = os.environ.get(GITHUB_API_TOKEN_NAME) if not github_api_token: - print "Error: GitHub API Key is Required!" + print("Error: GitHub API Key is Required!") sys.exit(1) session = requests.Session() @@ -174,7 +210,7 @@ def repo_is_included(issue, repo_matcher, repo_excluder): if not repo_is_included(issue, repo_matcher, repo_excluder): continue -# print "{}:".format(issue['id']) +# print("{}:".format(issue['id'])) issue_author_id = issue['user']['id'] issue_author_login = issue['user']['login'] @@ -259,40 +295,40 @@ def repo_is_included(issue, repo_matcher, repo_excluder): closed_issue_author.append(issue) closed_issues[closed_issue_author_id] = closed_issue_author -print "=== Statistics for GitHub Organization '{0}' ====".format(github_org) +print("=== Statistics for GitHub Organization '{0}' ====".format(github_org)) -print "\n== General PR's ==\n" -for key, value in general_prs.iteritems(): +print("\n== General PR's ==\n") +for key, value in general_prs.items(): # Determine whether to print out Label if(show_label(key, input_labels)): if (human_readable): - print "{}:".format(key) - for label_key, label_value in value.iteritems(): + print("{}:".format(key)) + for label_key, label_value in value.items(): if (human_readable): - print " {0} - {1}".format(label_key, len(label_value)) + print(" {0} - {1}".format(label_key, len(label_value))) for issue_value in label_value: if (not human_readable): - print "Pull Requests/GH{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(issue_value['id'], label_key, 1, issue_value['repository_url'].split('/')[-2], issue_value['repository_url'].split('/')[-1], issue_value['number']) + print("Pull Requests/GH{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(issue_value['id'], label_key, 1, issue_value['repository_url'].split('/')[-2], issue_value['repository_url'].split('/')[-1], issue_value['number'])) else: - print " {0} - {1}".format(encode_text(issue_value['repository_url'].split('/')[-1]), encode_text(issue_value['title'])) + print(" {0} - {1}".format(encode_text(issue_value['repository_url'].split('/')[-1]), encode_text(issue_value['title']))) -print "\n== Reviewed PR's ==\n" -for key, value in reviewed_prs.iteritems(): +print("\n== Reviewed PR's ==\n") +for key, value in reviewed_prs.items(): if (not human_readable): - for issue_key, issue_value in value.iteritems(): - print "Reviewed Pull Requests/GH{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(issue_value['id'], key, 1, issue_value['repository_url'].split('/')[-2], issue_value['repository_url'].split('/')[-1], issue_value['number']) + for issue_key, issue_value in value.items(): + print("Reviewed Pull Requests/GH{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(issue_value['id'], key, 1, issue_value['repository_url'].split('/')[-2], issue_value['repository_url'].split('/')[-1], issue_value['number'])) else: - print "{0} - {1}".format(key, len(value)) - for issue_key, issue_value in value.iteritems(): - print " {0} - {1}".format(encode_text(issue_value['repository_url'].split('/')[-1]), encode_text(issue_value['title'])) + print("{0} - {1}".format(key, len(value))) + for issue_key, issue_value in value.items(): + print(" {0} - {1}".format(encode_text(issue_value['repository_url'].split('/')[-1]), encode_text(issue_value['title']))) -print "\n== Closed Issues ==\n" -for key, value in closed_issues.iteritems(): +print("\n== Closed Issues ==\n") +for key, value in closed_issues.items(): if (not human_readable): - print "Closed Issues/GH{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(key, value[0]['assignee']['login'], len(value), value[0]['repository_url'].split('/')[-2], value[0]['repository_url'].split('/')[-1], value[0]['number']) + print("Closed Issues/GH{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(key, value[0]['assignee']['login'], len(value), value[0]['repository_url'].split('/')[-2], value[0]['repository_url'].split('/')[-1], value[0]['number'])) else: - print "{0} - {1}".format(value[0]['assignee']['login'], len(value)) + print("{0} - {1}".format(value[0]['assignee']['login'], len(value))) for issue_value in value: - print " {0} - {1}".format(encode_text(value['repository_url'].split('/')[-1]), encode_text(value[0]['title'])) + print(" {0} - {1}".format(encode_text(value['repository_url'].split('/')[-1]), encode_text(value[0]['title']))) diff --git a/gitlab-stats.py b/gitlab-stats.py index 326b524..b81a9ed 100755 --- a/gitlab-stats.py +++ b/gitlab-stats.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os import json @@ -7,7 +7,7 @@ import pytz import argparse import dateutil.parser -import urllib +import urllib.parse import re from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta @@ -30,8 +30,8 @@ def encode_text(text): if text: - return text.encode("utf-8") - + # In Python 3, strings are already unicode, so just return as-is + return text return text @@ -55,23 +55,23 @@ def valid_date(s): def handle_pagination_items(session, url): if is_debug: - print "DEBUG:: handle_pagination_items(): url = {0}".format(url) + print("DEBUG:: handle_pagination_items(): url = {0}".format(url)) pagination_request = session.get(url) pagination_request.raise_for_status() - if 'next' in pagination_request.headers["Link"] and pagination_request.links['next']: + if 'next' in pagination_request.headers.get("Link", "") and pagination_request.links.get('next'): return pagination_request.json() + handle_pagination_items(session, pagination_request.links['next']['url']) else: return pagination_request.json() def get_group(session, server, group_name): - group = session.get("{0}/api/v4/groups/{1}".format(server, urllib.quote(group_name, safe=''))) + group = session.get("{0}/api/v4/groups/{1}".format(server, urllib.parse.quote(group_name, safe=''))) global req_group result = group.json() if is_debug: - print "DEBUG:: Group Data" - print " {0}".format(json.dumps(result, indent=4, sort_keys=True)) + print("DEBUG:: Group Data") + print(" {0}".format(json.dumps(result, indent=4, sort_keys=True))) return result @@ -82,8 +82,8 @@ def get_project(session, project_id): project_cache[project_id]=project_request.json() if is_debug: - print "DEBUG:: Added project data to cache" - print " {0}".format(json.dumps(project_cache[project_id], indent=4, sort_keys=True)) + print("DEBUG:: Added project data to cache") + print(" {0}".format(json.dumps(project_cache[project_id], indent=4, sort_keys=True))) else: project_cache.get(project_id) @@ -93,12 +93,12 @@ def is_data_item_allowed(item, group, session, repo_matcher): include_item = False project = get_project(session, item["project_id"]) - project_is_org_child = re.match("^{0}\/".format(group["path"]), project["path_with_namespace"]) != None - item_matches = re.match(repo_matcher, project["path_with_namespace"]) != None + project_is_org_child = re.match("^{0}\/".format(group["path"]), project["path_with_namespace"]) is not None + item_matches = repo_matcher.match(project["path_with_namespace"]) is not None if project_is_org_child and item_matches: if is_debug: - print "DEBUG:: Including item - {0}".format(item["references"]["full"]) + print("DEBUG:: Including item - {0}".format(item["references"]["full"])) include_item = True return include_item @@ -109,7 +109,7 @@ def get_group_project_data(data_type, session, server, group, start_date, repo_m base_url = "{0}/api/v4/groups/{1}/{2}".format(server, group["id"], data_type) if is_debug: - print "DEBUG:: Getting {0} group {1}".format(group["path"], data_type) + print("DEBUG:: Getting {0} group {1}".format(group["path"], data_type)) query_state = "&state=" if data_type == "issues": @@ -124,7 +124,7 @@ def get_group_project_data(data_type, session, server, group, start_date, repo_m query_string = "?scope=all&per_page=1000{0}{1}".format(query_state, query_date) if is_debug: - print "DEBUG:: Query URL: {0}".format(base_url+query_string) + print("DEBUG:: Query URL: {0}".format(base_url+query_string)) query_result = handle_pagination_items(session, base_url+query_string) @@ -133,7 +133,7 @@ def get_group_project_data(data_type, session, server, group, start_date, repo_m allowed_data.append(item) if is_debug: - print "DEBUG:: ALLOWED_DATA - {0}\n{1}".format(data_type, json.dumps(allowed_data, indent=4, sort_keys=True)) + print("DEBUG:: ALLOWED_DATA - {0}\n{1}".format(data_type, json.dumps(allowed_data, indent=4, sort_keys=True))) return allowed_data @@ -162,7 +162,7 @@ def get_group_project_data(data_type, session, server, group, start_date, repo_m gitlab_server = os.getenv(GITLAB_SERVER_NAME, GITLAB_SERVER_DEFAULT) if not gitlab_api_token: - print "Error: GitLab API Token is Required!" + print("Error: GitLab API Token is Required!") sys.exit(1) session = requests.Session() @@ -174,7 +174,7 @@ def get_group_project_data(data_type, session, server, group, start_date, repo_m group = get_group(session, gitlab_server, gitlab_group) if group is None: - print "Unable to Locate Group!" + print("Unable to Locate Group!") sys.exit(1) @@ -187,10 +187,10 @@ def get_group_project_data(data_type, session, server, group, start_date, repo_m if dateutil.parser.parse(mr["merged_at"]) < start_date: if is_debug: - print "DEBUG:: Omit {0} MR {1} {2}/{3}".format(mr["state"], mr["merged_at"], mr['id'], mr['title']) + print("DEBUG:: Omit {0} MR {1} {2}/{3}".format(mr["state"], mr["merged_at"], mr['id'], mr['title'])) continue if is_debug: - print "DEBUG:: Incl {0} MR {1} {2}/{3}".format(mr["state"], mr["merged_at"], mr['id'], mr['title']) + print("DEBUG:: Incl {0} MR {1} {2}/{3}".format(mr["state"], mr["merged_at"], mr['id'], mr['title'])) # Filter out unwanted mr users (if username is specified, then we're only interested in MRs that have that user either the author or merger) if username is not None and (mr["author"]["username"] != username or mr["merged_by"]["username"] != username): @@ -198,7 +198,7 @@ def get_group_project_data(data_type, session, server, group, start_date, repo_m # Filter out if merged == author if mr["author"]["username"] == mr["merged_by"]["username"]: - print "# Error: Author==Merged_by {0} {1} {2}".format(mr['id'], mr["author"]["username"], mr['title']) + print("# Error: Author==Merged_by {0} {1} {2}".format(mr['id'], mr["author"]["username"], mr['title'])) continue # Merged MRs @@ -227,10 +227,10 @@ def get_group_project_data(data_type, session, server, group, start_date, repo_m if dateutil.parser.parse(iss["closed_at"]) < start_date: if is_debug: - print "DEBUG:: Omit {0} Issue {1} {2}/{3} (shortId={4})".format(iss["state"], iss["closed_at"], iss['id'], iss['title'], iss['iid']) + print("DEBUG:: Omit {0} Issue {1} {2}/{3} (shortId={4})".format(iss["state"], iss["closed_at"], iss['id'], iss['title'], iss['iid'])) continue if is_debug: - print "DEBUG:: Incl {0} Issue {1} {2}/{3} (shortId={4})".format(iss["state"], iss["closed_at"], iss['id'], iss['title'], iss['iid']) + print("DEBUG:: Incl {0} Issue {1} {2}/{3} (shortId={4})".format(iss["state"], iss["closed_at"], iss['id'], iss['title'], iss['iid'])) # Filter out if closed_by == author if iss["author"]["username"] == iss["closed_by"]["username"]: @@ -242,7 +242,7 @@ def get_group_project_data(data_type, session, server, group, start_date, repo_m # Filter out unwanted users if username is not None and (iss["author"]["username"] != username or iss["closed_by"]["username"] != username): - print "# Info: Filtered out : Issue was opened by {0}, and closed by {1}. User {2} was specified as filter".format(iss["author"]["username"], iss["closed_by"]["username"], username) + print("# Info: Filtered out : Issue was opened by {0}, and closed by {1}. User {2} was specified as filter".format(iss["author"]["username"], iss["closed_by"]["username"], username)) continue # Closed Issues @@ -254,45 +254,45 @@ def get_group_project_data(data_type, session, server, group, start_date, repo_m closed_issues[iss["closed_by"]["username"]] = closed_by_iss -print "=== Statistics for GitLab Group '{0}' ====".format(gitlab_group) +print("=== Statistics for GitLab Group '{0}' ====".format(gitlab_group)) -print "\n== Merged MR's ==\n" -for key, value in merged_mrs.iteritems(): +print("\n== Merged MR's ==\n") +for key, value in merged_mrs.items(): if human_readable: - print "{0} - {1}".format(value[0]["author"]["username"], len(value)) + print("{0} - {1}".format(value[0]["author"]["username"], len(value))) for mr_value in value: if not human_readable: # 1 point to author for opening a merged MR - print "Merge Requests/GL{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(mr_value['id'], mr_value['author']['username'], 1, mr_value['web_url'].split('/')[3], '/'.join(mr_value['web_url'].split('/')[4:(len(mr_value['web_url'].split('/'))-3)]), mr_value['web_url'].split('/')[-1]) + print("Merge Requests/GL{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(mr_value['id'], mr_value['author']['username'], 1, mr_value['web_url'].split('/')[3], '/'.join(mr_value['web_url'].split('/')[4:(len(mr_value['web_url'].split('/'))-3)]), mr_value['web_url'].split('/')[-1])) if is_debug: - print " {0}".format(json.dumps(mr, indent=4, sort_keys=True)) + print(" {0}".format(json.dumps(mr_value, indent=4, sort_keys=True))) else: - print " {0} - {1}".format(encode_text(mr_value['web_url'].split('/')[-1]), encode_text(mr_value['title'])) + print(" {0} - {1}".format(encode_text(mr_value['web_url'].split('/')[-1]), encode_text(mr_value['title']))) -print "\n== Reviewed MR's ==\n" -for key, value in reviewed_mrs.iteritems(): +print("\n== Reviewed MR's ==\n") +for key, value in reviewed_mrs.items(): if human_readable: - print "{0} - {1}".format(value[0]['merged_by']['username'], len(value)) + print("{0} - {1}".format(value[0]['merged_by']['username'], len(value))) for mr_value in value: if not human_readable: # 1 point to reviewer (assuming merged_by is reviewer) for merged MR's - print "Reviewed Merge Requests/GL{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(mr_value['id'], mr_value['merged_by']['username'], 1, mr_value['web_url'].split('/')[3], '/'.join(mr_value['web_url'].split('/')[4:(len(mr_value['web_url'].split('/'))-3)]), mr_value['web_url'].split('/')[-1]) + print("Reviewed Merge Requests/GL{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(mr_value['id'], mr_value['merged_by']['username'], 1, mr_value['web_url'].split('/')[3], '/'.join(mr_value['web_url'].split('/')[4:(len(mr_value['web_url'].split('/'))-3)]), mr_value['web_url'].split('/')[-1])) if is_debug: - print " {0}".format(json.dumps(mr_value, indent=4, sort_keys=True)) + print(" {0}".format(json.dumps(mr_value, indent=4, sort_keys=True))) else: - print " {0} - {1}".format(encode_text(mr_value['web_url'].split('/')[-1]), encode_text(mr_value['title'])) + print(" {0} - {1}".format(encode_text(mr_value['web_url'].split('/')[-1]), encode_text(mr_value['title']))) -print "\n== Closed Issues ==\n" -for key, value in closed_issues.iteritems(): +print("\n== Closed Issues ==\n") +for key, value in closed_issues.items(): if human_readable: - print "{0} - {1}".format(value[0]['closed_by']['username'], len(value)) + print("{0} - {1}".format(value[0]['closed_by']['username'], len(value))) for iss_value in value: if not human_readable: # 1 point person who closes an issue - print "Closed Issues/GL{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(iss_value['id'], iss_value['closed_by']['username'], 1, iss_value['web_url'].split('/')[3], '/'.join(iss_value['web_url'].split('/')[4:(len(iss_value['web_url'].split('/'))-3)]), iss_value['web_url'].split('/')[-1]) + print("Closed Issues/GL{0}/{1}/{2} [org={3}, board={4}, linkId={5}]".format(iss_value['id'], iss_value['closed_by']['username'], 1, iss_value['web_url'].split('/')[3], '/'.join(iss_value['web_url'].split('/')[4:(len(iss_value['web_url'].split('/'))-3)]), iss_value['web_url'].split('/')[-1])) if is_debug: - print " {0}".format(json.dumps(iss_value, indent=4, sort_keys=True)) + print(" {0}".format(json.dumps(iss_value, indent=4, sort_keys=True))) else: - print " {0} - {1}".format(encode_text(iss_value['web_url'].split('/')[-1]), encode_text(iss_value['title'])) + print(" {0} - {1}".format(encode_text(iss_value['web_url'].split('/')[-1]), encode_text(iss_value['title']))) diff --git a/hangouts-chat.py b/hangouts-chat.py index 70a1fd6..58cbb92 100755 --- a/hangouts-chat.py +++ b/hangouts-chat.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from oauth2client.service_account import ServiceAccountCredentials from os import path @@ -73,8 +73,8 @@ def handle_pagination_items(session, url, key, next_page_token=None): def encode_text(text): if text: - return text.encode("utf-8") - + # In Python 3, strings are already unicode, so just return as-is + return text return text parser = argparse.ArgumentParser(description='Gather Google Hangouts Statistics.') @@ -86,11 +86,11 @@ def encode_text(text): service_account_key_file = os.environ.get(SERVICE_ACCOUNT_KEY_FILE_NAME) if not service_account_key_file: - print "Error: Service Account Key File Location is Required!" + print("Error: Service Account Key File Location is Required!") sys.exit(1) if not path.exists(service_account_key_file): - print "Error: Service Account Key File Does Not Exist!" + print("Error: Service Account Key File Does Not Exist!") sys.exit(1) session = requests.Session() @@ -98,16 +98,16 @@ def encode_text(text): error = login(session, service_account_key_file) if error is not None: - print error + print(error) sys.exit(1) spaces_with_members = get_spaces_with_members(session) -print "=== Statistics for Google Hangouts Chat\n" +print("=== Statistics for Google Hangouts Chat\n") -for key, value in spaces_with_members.iteritems(): - print "- {0} - {1} Members".format(encode_text(value["space"]["displayName"]), len(value["members"])) +for key, value in spaces_with_members.items(): + print("- {0} - {1} Members".format(encode_text(value["space"]["displayName"]), len(value["members"]))) if show_members is not None: for member in value["members"]: - print " - {0}".format(encode_text(member["member"]["displayName"])) \ No newline at end of file + print(" - {0}".format(encode_text(member["member"]["displayName"]))) \ No newline at end of file diff --git a/mailman-subscribers.py b/mailman-subscribers.py index d1619f9..1d5c764 100755 --- a/mailman-subscribers.py +++ b/mailman-subscribers.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vi: set et sw=4 st=4: # # 2004-08-27 Jim Tittsler @@ -156,60 +156,44 @@ Tested with the Mailman 2.1.5 - 2.1.29 Membership list layout, but the --unhide option only works up to 2.1.22. - If Python 2.4's cookielib is available, use it. Otherwise require - ClientCookie http://wwwsearch.sourceforge.net/ClientCookie/ - - This script runs on your workstation and requires that you have Python - installed. It works best with Python 2.4.x - through Python 2.7.x. See mailman-subscribers3.py for a Python 3 version. + This script runs on your workstation and requires Python 3. """ import sys import re import string -import urllib import getopt -import httplib -import urllib2 from time import sleep -from HTMLParser import HTMLParser -# if we have Python 2.4's cookielib, use it -try: - import cookielib - policy = cookielib.DefaultCookiePolicy(rfc2965 = True) - cookiejar = cookielib.CookieJar(policy) - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)).open -except ImportError: - import ClientCookie - # if this is a new ClientCookie, we need to turn on RFC2965 cookies - cookiejar = ClientCookie.CookieJar() - try: - cookiejar.set_policy(ClientCookie.DefaultCookiePolicy(rfc2965 = True)) - # install an opener that uses this policy - opener = ClientCookie.build_opener( - ClientCookie.HTTPCookieProcessor(cookiejar)) - ClientCookie.install_opener(opener) - except AttributeError: - # must be an old ClientCookie, which already accepts RFC2965 cookies - pass - opener = ClientCookie.urlopen +from html.parser import HTMLParser +from http.cookiejar import CookieJar, DefaultCookiePolicy +from http.client import InvalidURL +from urllib.request import build_opener, HTTPCookieProcessor +from urllib.error import URLError, HTTPError +from urllib import parse as urllib_parse + +# Python 3: use stdlib cookie jar and opener +policy = DefaultCookiePolicy(rfc2965=True) +cookiejar = CookieJar(policy) +_opener_obj = build_opener(HTTPCookieProcessor(cookiejar)) -PROGRAM = sys.argv[0] -try: - True, False -except NameError: - True = 1 - False = 0 +def opener(url, data=None): + """Open URL with optional POST data (dict). Data is form-encoded.""" + if data is not None: + data = urllib_parse.urlencode(data).encode('utf-8') + return _opener_obj.open(url, data=data) + + +PROGRAM = sys.argv[0] def usage(code, msg=''): if code: fd = sys.stderr else: fd = sys.stdout - print >> fd, __doc__ % globals() + print(__doc__ % globals(), file=fd) if msg: - print >> fd, msg + print(msg, file=fd) sys.exit(code) subscribers = {} @@ -233,17 +217,15 @@ def handle_starttag(self, tag, attrs): subemail = v[:-len(vname)] s = True elif a == 'value': - subval = v + subval = v or '' if s: - if not subscribers.has_key(subemail): + if subemail not in subscribers: subscribers[subemail] = {} if vname == '_nomail' and subval == "on": gotnomail = True else: - if not isinstance(subval, unicode): - subval = subval.decode(page_cset, 'replace') - subscribers[subemail][vname] = subval.encode( - my_cset, 'replace') + # Python 3: HTMLParser gives str; store as-is + subscribers[subemail][vname] = subval if tag == 'a': for a,v in attrs: if a == 'href' and v.find("%s/" % (url_path)) >= 0: @@ -288,7 +270,7 @@ def main(): if o in ("-h", "--help"): usage(0) if o in ("-o", "--output"): - fp = open(a, "wt") + fp = open(a, "w", errors='replace') if o in ("-f", "--fullnames"): fullnames = True if o in ("-n", "--nomail"): @@ -329,9 +311,9 @@ def main(): # login, picking up the cookie try: - page = opener(member_url, urllib.urlencode(p)) - except (urllib2.URLError, httplib.InvalidURL), e: - if isinstance(e, urllib2.HTTPError) and e.code == 401: + page = opener(member_url, p) + except (URLError, InvalidURL) as e: + if isinstance(e, HTTPError) and e.code == 401: usage(1, 'Invalid password.') else: usage(1, """Error accessing %s @@ -340,12 +322,13 @@ def main(): """ % (member_url)) # Get the charset of the page, but use iso-8859-1 for ascii or None. - page_cset = page.info().getparam('charset') or 'iso-8859-1' - if page_cset.lower().endswith('ascii'): + page_cset = page.headers.get_content_charset() or 'iso-8859-1' + if page_cset and page_cset.lower().endswith('ascii'): page_cset = 'iso-8859-1' - lines = page.read() + raw = page.read() page.close() + lines = raw.decode(page_cset, 'replace') p = {} # Try to recognize the returned page independent of the list language if re.search(r'INPUT\s+type="SUBMIT"\s+name="admlogin"', lines, @@ -369,18 +352,17 @@ def main(): maxchunk = 0 while chunk <= maxchunk: if verbose: - print >> sys.stderr, "%c(%d)" % (letter, chunk) + print("%c(%d)" % (letter, chunk), file=sys.stderr) while True: try: page = opener(member_url + "?letter=%s&chunk=%d" % (letter, chunk)) - lines = page.read() + raw = page.read() page.close() - except urllib2.URLError: + lines = raw.decode(page_cset, 'replace') + except URLError: if verbose: - print >> sys.stderr,\ - 'Error encountered in accessing web page.',\ - 'Retrying.' + print('Error encountered in accessing web page. Retrying.', file=sys.stderr) sleep(2) else: break @@ -390,44 +372,41 @@ def main(): parser.close() chunk += 1 - subscriberlist = subscribers.items() - subscriberlist.sort() + subscriberlist = sorted(subscribers.items()) # print the subscribers list if csv: - print >>fp, '"Full name","email address","mod","hide",\ -"nomail","ack","not metoo","nodupes","digest","plain"' + print('"Full name","email address","mod","hide",' + '"nomail","ack","not metoo","nodupes","digest","plain"', file=fp) nunhide = 0 for (email, d) in subscriberlist: if unhide and d['_hide'] == "on": - params = urllib.urlencode({'conceal':0, - 'options-submit':1}) + params = {'conceal': 0, 'options-submit': 1} u = opener("%s/%s" % (options_url, email), params) u.close() d['_hide'] = "off" nunhide += 1 if verbose and nunhide % 100 == 0: - print >>sys.stderr, '.', - email = urllib.unquote(email) + print('.', end='', file=sys.stderr) + email = urllib_parse.unquote(email) if csv: - print >>fp,\ - '"%s","%s","%s","%s","%s","%s","%s","%s","%s","%s"'\ - % (d['_realname'], email, d['_mod'], d['_hide'], - d['_nomail'], d['_ack'], d['_notmetoo'], - d['_nodupes'], d['_digest'], d['_plain']) + print('"%s","%s","%s","%s","%s","%s","%s","%s","%s","%s"' % + (d['_realname'], email, d['_mod'], d['_hide'], + d['_nomail'], d['_ack'], d['_notmetoo'], + d['_nodupes'], d['_digest'], d['_plain']), file=fp) continue - if nomail == 'enabled' and d['_nomail'] <> "off": + if nomail == 'enabled' and d['_nomail'] != "off": continue if nomail == 'any' and d['_nomail'] == "off": continue - if nomail == 'admin' and d['_nomail'] <> "[A]": + if nomail == 'admin' and d['_nomail'] != "[A]": continue - if nomail == 'bounce' and d['_nomail'] <> "[B]": + if nomail == 'bounce' and d['_nomail'] != "[B]": continue - if nomail == 'user' and d['_nomail'] <> "[U]": + if nomail == 'user' and d['_nomail'] != "[U]": continue - if nomail == 'unknown' and d['_nomail'] <> "[?]": + if nomail == 'unknown' and d['_nomail'] != "[?]": continue if regular and d['_digest'] == "on": continue @@ -438,9 +417,9 @@ def main(): if digest == "plain" and d['_plain'] == "off": continue if not fullnames or d['_realname'] == "": - print >>fp, email + print(email, file=fp) else: - print >>fp, '%s <%s>' % (d['_realname'], email) + print('%s <%s>' % (d['_realname'], email), file=fp) fp.close() diff --git a/requirements.txt b/requirements.txt index 8a61e6d..e1eb58c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -requests +oauth2client python-dateutil pytz -google-api-python-client \ No newline at end of file +requests +smartsheet-python-sdk diff --git a/rocketchat.py b/rocketchat.py index 803e591..e26ea9b 100755 --- a/rocketchat.py +++ b/rocketchat.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import os, json, requests, sys, argparse, collections, re, operator, csv +import os, json, requests, sys, argparse, collections.abc, re, operator, csv from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta @@ -85,7 +85,7 @@ def process_item(final_dict, history_type, key): final_dict['statistics'][history_type] += 1 def plural_items(text, obj): - if obj is not None and (isinstance(obj, collections.Iterable) and len(obj) == 1) or obj == 1: + if obj is not None and (isinstance(obj, collections.abc.Iterable) and len(obj) == 1) or obj == 1: return text[:-1] else: return text @@ -167,19 +167,19 @@ def write_ouput_file_record(filename, output_file_records, first_record=None): error = login(session, server, rocketchat_username, rocketchat_password, rocketchat_auth_token, rocketchat_user_id) if error is not None: - print error + print(error) sys.exit(1) channels = get_channels(session, server) filter_channels(channels, filtered_text) -newest_date = datetime.now().utcnow() +newest_date = datetime.utcnow() oldest_date = newest_date - relativedelta(days=days) formatted_time_period = "{0} - {1}".format(oldest_date.strftime("%m/%d/%Y"), newest_date.strftime("%m/%d/%Y")) -print "=== Rocketchat Statistics For {0} ===\n".format(formatted_time_period) +print("=== Rocketchat Statistics For {0} ===\n".format(formatted_time_period)) if len(channels) > 0: for channel_index, channel in enumerate(channels): @@ -192,20 +192,20 @@ def write_ouput_file_record(filename, output_file_records, first_record=None): users_removed = channel_history_stats['statistics']['removed'] total_messages = channel_history_stats['statistics']['messages'] - print formatted_channel_name - print " {0} {1} Joined".format(users_joined, plural_items("Users", users_joined)) - print " {0} {1} Removed".format(users_removed, plural_items("Users", users_removed)) - print " {0} {1}".format(total_messages, plural_items("Messages", total_messages)) + print(formatted_channel_name) + print(" {0} {1} Joined".format(users_joined, plural_items("Users", users_joined))) + print(" {0} {1} Removed".format(users_removed, plural_items("Users", users_removed))) + print(" {0} {1}".format(total_messages, plural_items("Messages", total_messages))) output_file_user_messages = "" - for username, username_num_messages in sorted(channel_history_stats['messages'].iteritems(), key=lambda (k,v): (v,k), reverse=True): + for username, username_num_messages in sorted(channel_history_stats['messages'].items(), key=lambda kv: (kv[1], kv[0]), reverse=True): user_messages = "{0} - {1:.2f}% - {2} {3}".format(username, (float(username_num_messages)/float(total_messages)*100), username_num_messages, plural_items("Messages", username_num_messages)) - print " * {0}".format(user_messages) + print(" * {0}".format(user_messages)) - if output_file_user_messages is not "": + if output_file_user_messages != "": output_file_user_messages += "\n" output_file_user_messages += user_messages @@ -224,4 +224,4 @@ def write_ouput_file_record(filename, output_file_records, first_record=None): write_ouput_file_record(output_file, output_file_row_records) else: - print "No Rocketchat Channels Match the description '{0}'".format(filtered_text) + print("No Rocketchat Channels Match the description '{0}'".format(filtered_text)) diff --git a/smartsheets-reports-stats.py b/smartsheets-reports-stats.py index 28e9735..7d7ca3d 100755 --- a/smartsheets-reports-stats.py +++ b/smartsheets-reports-stats.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import json,argparse,sys,re,os,requests from datetime import datetime, timedelta @@ -28,15 +28,15 @@ def valid_date(s): board_id = args.board_id if start_date is None: - print "Error: Please provide a start date!" + print("Error: Please provide a start date!") sys.exit(1) if sheet_id is None: - print "Error: Smartsheets sheet ID must be provided!" + print("Error: Smartsheets sheet ID must be provided!") sys.exit(1) if board_id is None: - print "Error: Smartsheets board ID must be provided in order to build a like back to the origin of the points!" + print("Error: Smartsheets board ID must be provided in order to build a like back to the origin of the points!") sys.exit(1) if points_grouping is None: @@ -45,7 +45,7 @@ def valid_date(s): api_token = os.environ.get(API_TOKEN_NAME) if not api_token: - print "Error: Smartsheets API Key is Required!" + print("Error: Smartsheets API Key is Required!") sys.exit(1) @@ -91,10 +91,10 @@ def get_cell_by_column_name(row, column_name): else: pool="ServicesSupport" - print "{0}/SS{1}/{2}/{3} [pool={4},board={5},rowId={6},linkId={7}]".format(points_grouping, row["id"], recipient,int(row["Points"]),pool,board_id,row["id"],row["Row ID"]) + print("{0}/SS{1}/{2}/{3} [pool={4},board={5},rowId={6},linkId={7}]".format(points_grouping, row["id"], recipient,int(row["Points"]),pool,board_id,row["id"],row["Row ID"])) # outputs Giveback "duplicate records" as output. Used to prevent historical duplicate allocation of points - #print "\"SS{0}.{1}\",".format(row["id"],recipient) + #print("\"SS{0}.{1}\",".format(row["id"],recipient)) diff --git a/smartsheets-stats.py b/smartsheets-stats.py index 6adce4b..e61efe1 100644 --- a/smartsheets-stats.py +++ b/smartsheets-stats.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import smartsheet,json,argparse,sys,re,os from datetime import datetime, timedelta @@ -28,15 +28,15 @@ def valid_date(s): board_id = args.board_id if start_date is None: - print "Error: Please provide a start date!" + print("Error: Please provide a start date!") sys.exit(1) if sheet_id is None: - print "Error: Smartsheets sheet ID must be provided!" + print("Error: Smartsheets sheet ID must be provided!") sys.exit(1) if board_id is None: - print "Error: Smartsheets board ID must be provided in order to build a like back to the origin of the points!" + print("Error: Smartsheets board ID must be provided in order to build a like back to the origin of the points!") sys.exit(1) if points_grouping is None: @@ -45,7 +45,7 @@ def valid_date(s): api_token = os.environ.get(API_TOKEN_NAME) if not api_token: - print "Error: Smartsheets API Key is Required!" + print("Error: Smartsheets API Key is Required!") sys.exit(1) @@ -91,10 +91,10 @@ def get_cell_by_column_name(row, column_name): if re.search("First and Thirds.*", row["Program Name"]): pool="ServicesSupport" - print "{0}/SS{1}/{2}/{3} [pool={4},board={5},rowId={6},linkId={7}]".format(points_grouping, row["id"], recipient,int(row["Points"]),pool,board_id,row["id"],row["Row ID"]) + print("{0}/SS{1}/{2}/{3} [pool={4},board={5},rowId={6},linkId={7}]".format(points_grouping, row["id"], recipient,int(row["Points"]),pool,board_id,row["id"],row["Row ID"])) # outputs Giveback "duplicate records" as output. Used to prevent historical duplicate allocation of points - #print "\"SS{0}.{1}\",".format(row["id"],recipient) + #print("\"SS{0}.{1}\",".format(row["id"],recipient)) diff --git a/trello-stats.py b/trello-stats.py index 548ebbd..fca0c63 100755 --- a/trello-stats.py +++ b/trello-stats.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import os, json, requests, sys, argparse, collections, re +import os, json, requests, sys, argparse, collections.abc, re from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta @@ -64,11 +64,11 @@ def get_member(session, member_id): requestCount_member+=1 member_request.raise_for_status() memberCache[member_id]=member_request.json() - if debug: print "get_member:: memberCache.add({0})".format(memberCache[member_id]['username']) + if debug: print("get_member:: memberCache.add({0})".format(memberCache[member_id]['username'])) return memberCache.get(member_id) def plural_items(text, obj): - if obj is not None and (isinstance(obj, collections.Iterable) and len(obj) == 1) or obj == 1: + if obj is not None and (isinstance(obj, collections.abc.Iterable) and len(obj) == 1) or obj == 1: return text[:-1] else: return text @@ -83,8 +83,8 @@ def calculate_points(text): def encode_text(text): if text: - return text.encode("utf-8") - + # In Python 3, strings are already unicode, so just return as-is + return text return text def preload_member_cache_from_org(session, org_id): @@ -108,7 +108,7 @@ def preload_member_cache_from_board(session, board_id): def add_member_to_cache(member): if member['id'] not in memberCache: - if debug: print "add_member_to_cache:: memberCache.add({0})".format(member['username']) + if debug: print("add_member_to_cache:: memberCache.add({0})".format(member['username'])) memberCache[member['id']] = {"id":member['id'], "username":member['username'], "fullName":member['fullName']} @@ -118,7 +118,7 @@ def add_member_to_cache(member): if not trello_api_key or not trello_api_token: - print "Error: Trello API Key and API Token are Required!" + print("Error: Trello API Key and API Token are Required!") sys.exit(1) parser = argparse.ArgumentParser(description='Gather Trello Statistics.') @@ -193,12 +193,12 @@ def add_member_to_cache(member): members_items[member_id] = member_items if (not human_readable): - print "{0}/TR{1}/{2}/{3} [linkId={4},board={5}]".format(points_grouping, card_id, get_member(session, member_id)['username'], points, card['shortLink'], card['board']['name']) + print("{0}/TR{1}/{2}/{3} [linkId={4},board={5}]".format(points_grouping, card_id, get_member(session, member_id)['username'], points, card['shortLink'], card['board']['name'])) if (human_readable): - print "=== Statistics for Trello Team '{0}' ====\n".format(encode_text(org_response['displayName']) if 'displayName' in org_response else encode_text(org_response['name'])) - for key, value in members_items.iteritems(): + print("=== Statistics for Trello Team '{0}' ====\n".format(encode_text(org_response['displayName']) if 'displayName' in org_response else encode_text(org_response['name']))) + for key, value in members_items.items(): member = get_member(session, key) value_points = value['points'] value_cards = value['cards'] @@ -206,8 +206,8 @@ def add_member_to_cache(member): if username is not None and member['username'] != username: continue - print "{0} has {1} {2} - {3} {4}".format(encode_text(member['username']), len(value_cards), plural_items("cards", value_cards), value_points, plural_items("points", value_points)) + print("{0} has {1} {2} - {3} {4}".format(encode_text(member['username']), len(value_cards), plural_items("cards", value_cards), value_points, plural_items("points", value_points))) for card in value['cards']: - print " - Board: {0} | Card: {1}".format(encode_text(cards[card]['board']['name']), encode_text(cards[card]['name'])) + print(" - Board: {0} | Card: {1}".format(encode_text(cards[card]['board']['name']), encode_text(cards[card]['name']))) -if debug: print "REQUESTS: org={0}, orgMembers={1}, member={2}, boardMembers={3}, cards={4}".format(requestCount_org, requestCount_orgMembers, requestCount_member, requestCount_boardMembers, requestCount_cards) +if debug: print("REQUESTS: org={0}, orgMembers={1}, member={2}, boardMembers={3}, cards={4}".format(requestCount_org, requestCount_orgMembers, requestCount_member, requestCount_boardMembers, requestCount_cards))