Skip to content

Parser fix #167

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
haraldschilly merged 3 commits into from
Oct 1, 2025
Merged

Parser fix #167

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e2dd392
Fix PEP 701 stuff
vincentmacri Oct 1, 2025
684b0a1
ruff format
vincentmacri Oct 1, 2025
4e6d4df
Remove unused import
vincentmacri Oct 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 69 additions & 63 deletions pubparse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@

# importing modules from Python library
import copy
import re
import os
import sys
from pprint import pprint
Expand Down Expand Up @@ -72,14 +71,14 @@
# the file containing the Sage-Combinat bibliography formatted in HTML
html_combinat = os.path.join(PWD, "publications-combinat.html")
# upstream version of the BibTeX database of Sage-Combinat
#bibtex_sage_combinat = "http://combinat.sagemath.org/hgwebdir.cgi/misc/raw-file/tip/articles/Sage-Combinat.bib"
# bibtex_sage_combinat = "http://combinat.sagemath.org/hgwebdir.cgi/misc/raw-file/tip/articles/Sage-Combinat.bib"
# the file containing the MuPAD-Combinat BibTeX database
publications_mupad = os.path.join(PWD, "MuPAD-Combinat.bib")
# the file containing the MuPAD publications list formatted in HTML
html_mupad = os.path.join(PWD, "publications-mupad.html")
# MathSciNet
publications_mathscinet = os.path.join(PWD, 'mathscinet.bib')
html_mathscinet = os.path.join(PWD, 'publications-mathscinet.html')
publications_mathscinet = os.path.join(PWD, "mathscinet.bib")
html_mathscinet = os.path.join(PWD, "publications-mathscinet.html")

# Stuff relating to file permissions.
# whether we should change the permissions of a file
Expand Down Expand Up @@ -316,17 +315,18 @@ def extract_publication(entry_dict):
for attribute in entry_dict.fields.keys():
publication_dict.setdefault(
str(attribute).strip().lower(),
unicode(entry_dict.fields[attribute]).strip())
unicode(entry_dict.fields[attribute]).strip(),
)
# The author field is a required field in BibTeX format.
# Extract author names.
authors_str = ""
authors_list = entry_dict.persons["author"]
authors_str = str(unicode(plain(authors_list[0]).format()))
if len(authors_list) > 1:
for author in authors_list[1:]:
authors_str = u"".join(
[authors_str, " and ",
str(unicode(plain(author).format()))])
authors_str = "".join(
[authors_str, " and ", str(unicode(plain(author).format()))]
)
authors_str = authors_str.replace("<nbsp>", " ")
publication_dict.setdefault("author", authors_str)
# The editor field is an optional field in BibTeX format.
Expand All @@ -337,10 +337,9 @@ def extract_publication(entry_dict):
editors_str = str(unicode(plain(editors_list[0]).format()))
if len(editors_list) > 1:
for editor in editors_list[1:]:
editors_str = u"".join([
editors_str, " and ",
str(unicode(plain(editor).format()))
])
editors_str = "".join(
[editors_str, " and ", str(unicode(plain(editor).format()))]
)
editors_str = editors_str.replace("<nbsp>", " ")
publication_dict.setdefault("editor", editors_str)
return publication_dict
Expand Down Expand Up @@ -370,10 +369,7 @@ def filter_undergraduate_theses(publications):
undergraduate_theses.append(item)
else:
preprints.append(item)
return {
"preprints": preprints,
"undergraduatetheses": undergraduate_theses
}
return {"preprints": preprints, "undergraduatetheses": undergraduate_theses}


def format_articles(articles):
Expand All @@ -400,17 +396,19 @@ def format_articles(articles):
try:
htmlstr = "".join([format_names(article["author"]), ". "])
htmlstr = "".join([htmlstr, html_title(article)])
aj = article.get("journal", article.get('journaltitle'))
aj = article.get("journal", article.get("journaltitle"))
htmlstr = "".join([htmlstr, aj, ", "])
for attribute in optional_attributes:
if attribute in article:
htmlstr = "".join(
[htmlstr, attribute, " ", article[attribute], ", "])
ay = article.get("year", article.get('date'))
[htmlstr, attribute, " ", article[attribute], ", "]
)
ay = article.get("year", article.get("date"))
htmlstr = "".join([htmlstr, ay, "."])
formatted_articles.append(htmlstr.strip())
except Exception as ex:
from pprint import pprint

pprint(article)
raise ex
return list(map(replace_special, formatted_articles))
Expand Down Expand Up @@ -472,14 +470,14 @@ def format_collections(collections):
htmlstr = "".join([htmlstr, html_title(entry)])
if "editor" in entry:
htmlstr = "".join(
[htmlstr, "In ",
format_names(entry["editor"]), " (ed.). "])
[htmlstr, "In ", format_names(entry["editor"]), " (ed.). "]
)
htmlstr = "".join([htmlstr, entry["booktitle"], ". "])
if "publisher" in entry:
htmlstr = "".join([htmlstr, entry["publisher"], ", "])
if "pages" in entry:
htmlstr = "".join([htmlstr, "pages ", entry["pages"], ", "])
ay = entry.get("year", entry.get('date'))
ay = entry.get("year", entry.get("date"))
htmlstr = "".join([htmlstr, ay, "."])
formatted_entries.append(htmlstr.strip())
return list(map(replace_special, formatted_entries))
Expand All @@ -500,7 +498,7 @@ def format_masterstheses(masterstheses):
A list of Master's theses all of which are formatted in HTML
suitable for displaying on websites.
"""
return format_theses(masterstheses, 'Masters thesis')
return format_theses(masterstheses, "Masters thesis")


def format_miscs(miscs, thesis=False):
Expand Down Expand Up @@ -536,9 +534,9 @@ def format_miscs(miscs, thesis=False):
note = entry["note"]
# handle the case: note = {<url> Bachelor thesis},
if "http://" in note:
note = note[note.find(" "):].strip()
note = note[note.find(" ") :].strip()
htmlstr = "".join([htmlstr, note, ", "])
y = entry.get("year", entry.get('date'))
y = entry.get("year", entry.get("date"))
htmlstr = "".join([htmlstr, y, "."])
formatted_miscs.append(htmlstr.strip())
except Exception as ex:
Expand Down Expand Up @@ -573,6 +571,7 @@ def format_names(names) -> str:
formatted_names[i] = "".join([formatted_names[i], ", "])
return "".join(formatted_names)


def format_theses(theses: list[dict[str, str]], thesis_type: str) -> list[str]:
r"""
Format each thesis in HTML format.
Expand All @@ -594,24 +593,23 @@ def format_theses(theses: list[dict[str, str]], thesis_type: str) -> list[str]:
formatted_theses: list[str] = []
for thesis in theses:
try:
htmlstr = ''.join([format_names(thesis['author']), '. '])
htmlstr = ''.join([htmlstr, html_title(thesis)])

htmlstr = ''.join([htmlstr, f'{thesis.get('type', thesis_type)}, '])
htmlstr = format_names(thesis["author"]) + ". "
htmlstr += html_title(thesis)

ts = thesis['school']
htmlstr = ''.join([htmlstr, ts, ', '])
htmlstr += thesis.get("type", thesis_type) + ", "
htmlstr += thesis["school"] + ", "

if 'address' in thesis:
htmlstr = ''.join([htmlstr, thesis['address'], ', '])
if "address" in thesis:
htmlstr += thesis["address"] + ", "

htmlstr = ''.join([htmlstr, thesis['year'], '.'])
htmlstr += thesis["year"] + "."
formatted_theses.append(htmlstr.strip())
except Exception as ex:
pprint(thesis)
raise ex
return list(map(replace_special, formatted_theses))


def format_phdtheses(phdtheses):
r"""
Format each PhD thesis in HTML format.
Expand All @@ -627,7 +625,7 @@ def format_phdtheses(phdtheses):
A list of PhD theses all of which are formatted in HTML
suitable for displaying on websites.
"""
return format_theses(phdtheses, 'PhD thesis')
return format_theses(phdtheses, "PhD thesis")


def format_techreports(techreports):
Expand Down Expand Up @@ -655,8 +653,9 @@ def format_techreports(techreports):
htmlstr = "".join([htmlstr, report["address"], ", "])
if "number" in report:
htmlstr = "".join(
[htmlstr, "technical report number ", report["number"], ", "])
y = report.get("year", report.get('date'))
[htmlstr, "technical report number ", report["number"], ", "]
)
y = report.get("year", report.get("date"))
htmlstr = "".join([htmlstr, y, "."])
formatted_reports.append(htmlstr.strip())
return list(map(replace_special, formatted_reports))
Expand Down Expand Up @@ -684,8 +683,8 @@ def format_proceedings(proceedings):
htmlstr = "".join([htmlstr, html_title(article)])
if "editor" in article:
htmlstr = "".join(
[htmlstr, "In ",
format_names(article["editor"]), " (ed.). "])
[htmlstr, "In ", format_names(article["editor"]), " (ed.). "]
)
htmlstr = "".join([htmlstr, article["booktitle"], ". "])
if "publisher" in article:
htmlstr = "".join([htmlstr, article["publisher"], ", "])
Expand All @@ -698,7 +697,7 @@ def format_proceedings(proceedings):
htmlstr = "".join([htmlstr, "Pages ", article["pages"], ", "])
else:
htmlstr = "".join([htmlstr, "pages ", article["pages"], ", "])
ay = article.get("year", article.get('date'))
ay = article.get("year", article.get("date"))
htmlstr = "".join([htmlstr, ay, "."])
formatted_proceedings.append(htmlstr.strip())
return list(map(replace_special, formatted_proceedings))
Expand Down Expand Up @@ -727,7 +726,7 @@ def format_unpublisheds(unpublisheds):
htmlstr = "".join([htmlstr, html_title(entry)])
if "month" in entry:
htmlstr = "".join([htmlstr, entry["month"], ", "])
y = entry.get('year', entry.get('date'))
y = entry.get("year", entry.get("date"))
htmlstr = "".join([htmlstr, y, "."])
formatted_entries.append(htmlstr.strip())
except Exception as ex:
Expand Down Expand Up @@ -764,7 +763,7 @@ def html_title(publication):
title = publication["title"]
if url != "":
if ("http://" in url) or ("https://" in url):
return "".join(["<a href=\"", url, "\">", title, "</a>", ". "])
return "".join(['<a href="', url, '">', title, "</a>", ". "])
# handle the case where no URL is provided or the "note" field doesn't
# contain a URL
return "".join([title, ". "])
Expand Down Expand Up @@ -818,27 +817,32 @@ def macro(name, sorted_index, papers):
# Sort the publication items. Journal articles, items in collections,
# and proceedings papers are grouped in one section. Sort these.
papers = articles + collections + proceedings + techreports
sorted_index = sort_publications(publications["articles"] +
publications["incollections"] +
publications["inproceedings"] +
publications["techreports"])
sorted_index = sort_publications(
publications["articles"]
+ publications["incollections"]
+ publications["inproceedings"]
+ publications["techreports"]
)
# insert the new list of articles
htmlcontent += macro("papers", sorted_index, papers)

# Sort the list of theses. These include PhD, Master's, and undergraduate
# theses.
theses = masterstheses + phdtheses + undergradtheses
sorted_index = sort_publications(publications["masterstheses"] +
publications["phdtheses"] +
miscs["undergraduatetheses"])
sorted_index = sort_publications(
publications["masterstheses"]
+ publications["phdtheses"]
+ miscs["undergraduatetheses"]
)
# insert the new list of theses
htmlcontent += macro("thesis", sorted_index, theses)

# Sort the list of books. These include both published books and
# unpublished manuscripts.
books_list = books + unpublisheds
sorted_index = sort_publications(publications["books"] +
publications["unpublisheds"])
sorted_index = sort_publications(
publications["books"] + publications["unpublisheds"]
)
htmlcontent += macro("books", sorted_index, books_list)

# Sort the list of preprints.
Expand Down Expand Up @@ -908,8 +912,9 @@ def process_database(dbfilename):
try:
pub_list.append(extract_publication(bibdb.entries[key]))
except Exception as ex:
#raise ex
# raise ex
import json

print(key)
print(json.dumps(bibdb.entries[key]))
raise ex
Expand All @@ -923,7 +928,7 @@ def process_database(dbfilename):
"miscs": misc,
"phdtheses": phdthesis,
"techreports": techreport,
"unpublisheds": unpublished
"unpublisheds": unpublished,
}


Expand Down Expand Up @@ -1108,14 +1113,13 @@ def sort_by_name(publications):
"""
NAME_INDEX = 0
POSITION_INDEX = 1
author_names = [(publications[i]["author"], i)
for i in range(len(publications))]
last_names = [(surname(author_names[i][NAME_INDEX]), i)
for i in range(len(author_names))]
author_names = [(publications[i]["author"], i) for i in range(len(publications))]
last_names = [
(surname(author_names[i][NAME_INDEX]), i) for i in range(len(author_names))
]
sorted_names = sorted(last_names)
return [
publications[sorted_names[i][POSITION_INDEX]]
for i in range(len(sorted_names))
publications[sorted_names[i][POSITION_INDEX]] for i in range(len(sorted_names))
]


Expand All @@ -1138,14 +1142,15 @@ def sort_by_year(publications):
pairs. Each publication year is a four-digit year. The publications list
contains items published during that year.
"""
item_years = [(publications[i].get("year", publications[i].get('date')), i)
for i in range(len(publications))]
item_years = [
(publications[i].get("year", publications[i].get("date")), i)
for i in range(len(publications))
]
sorted_years = sorted(item_years)
items_dict = {}
for year, item in sorted_years:
if year in items_dict:
items_dict.setdefault(year,
items_dict[year].append(publications[item]))
items_dict.setdefault(year, items_dict[year].append(publications[item]))
else:
items_dict.setdefault(year, [publications[item]])
return items_dict
Expand Down Expand Up @@ -1182,6 +1187,7 @@ def surname(name):
# os.system("rm " + publications_combinat)
# os.system("wget " + bibtex_sage_combinat)
import sys

if len(sys.argv) >= 2:
what = sys.argv[1]
else:
Expand Down