archive.js

const fs = require('fs');
const path = require('path');
const https = require('https');
const http = require('http');

const BIB_FILE = path.join(__dirname, 'cv.bib');
const PUB_DIR = path.join(__dirname, 'publications');
const README_FILE = path.join(__dirname, 'README.md');

function parseBibTeX(bibtexContent) {
    const stringDefs = extractStringDefinitions(bibtexContent);
    const rawEntries = extractEntries(bibtexContent);
    const entriesMap = new Map(rawEntries.map(e => [e.key, e]));
    const usedCrossrefs = new Set();

    const resolvedEntries = rawEntries.map(entry => {
        if (entry.fields.crossref) {
            const parentKey = entry.fields.crossref;
            const parent = entriesMap.get(parentKey);
            if (parent) {
                usedCrossrefs.add(parentKey);
                return {
                    ...entry,
                    fields: { ...parent.fields, ...entry.fields },
                    raw: entry.raw + '\n\n' + parent.raw
                };
            }
        }
        return entry;
    });

    return resolvedEntries
        .filter(entry => !usedCrossrefs.has(entry.key))
        .map(entry => normalizeEntry(entry, stringDefs))
        .filter(entry => entry !== null);
}

function extractStringDefinitions(content) {
    const defs = {};
    const stringPattern = /@string\s*\{\s*(\w+)\s*=\s*\{([^}]*)\}\s*\}/gi;
    let match;

    while ((match = stringPattern.exec(content)) !== null) {
        defs[match[1].toLowerCase()] = match[2].trim();
    }

    return defs;
}

function extractEntries(content) {
    const entries = [];
    const entryPattern = /@(\w+)\s*\{\s*([^,\s]+)\s*,/g;
    let match;
    let index = 0;

    while ((match = entryPattern.exec(content)) !== null) {
        const type = match[1].toLowerCase();
        const key = match[2];
        const startPos = match.index;

        if (['preamble', 'string', 'comment'].includes(type)) continue;

        const { fieldsContent, rawContent } = extractEntryContent(content, startPos);
        if (fieldsContent) {
            const fields = parseFields(fieldsContent);
            entries.push({ type, key, fields, raw: rawContent, index: index++ });
        }
    }

    return entries;
}

function extractEntryContent(content, startPos) {
    let braceCount = 0;
    let pos = startPos;
    let foundFirstBrace = false;
    let fieldsStart = -1;

    while (pos < content.length) {
        if (content[pos] === '{') {
            if (!foundFirstBrace) {
                foundFirstBrace = true;
                fieldsStart = pos + 1;
            }
            braceCount++;
        } else if (content[pos] === '}') {
            braceCount--;
            if (foundFirstBrace && braceCount === 0) {
                return {
                    fieldsContent: content.slice(fieldsStart, pos),
                    rawContent: content.slice(startPos, pos + 1)
                };
            }
        }
        pos++;
    }

    return { fieldsContent: null, rawContent: null };
}

function parseFields(content) {
    const fields = {};
    const fieldPattern = /(\w+)\s*=\s*(?:\{([^{}]*(?:\{[^{}]*\}[^{}]*)*)\}|"([^"]*)"|(\w+))/g;
    let match;

    while ((match = fieldPattern.exec(content)) !== null) {
        const key = match[1].toLowerCase();
        const value = match[2] || match[3] || match[4] || '';
        fields[key] = key === 'note' ? value.trim() : cleanLatex(value.trim());
    }

    return fields;
}

function cleanLatex(text) {
    if (!text) return '';
    return text
        .replace(/\$\^[\{]?([^\$\}]+)[\}]?\$/g, '$1')
        .replace(/\^\{([^}]+)\}/g, '$1')
        .replace(/\$_[\{]?([^\$\}]+)[\}]?\$/g, '$1')
        .replace(/_\{([^}]+)\}/g, '$1')
        .replace(/\\href\{([^}]*)\}\{([^}]*)\}/g, '$2')
        .replace(/\\url\{([^}]*)\}/g, '$1')
        .replace(/\\&/g, '&')
        .replace(/\\\\/g, '')
        .replace(/\\'/g, "'")
        .replace(/\\"/g, '"')
        .replace(/\\`/g, '`')
        .replace(/\\~/g, '~')
        .replace(/\\textit\{([^}]*)\}/g, '$1')
        .replace(/\\textbf\{([^}]*)\}/g, '$1')
        .replace(/\\emph\{([^}]*)\}/g, '$1')
        .replace(/[\{\}]/g, '')
        .replace(/\$/g, '')
        .replace(/\\coe/g, '')
        .replace(/\s+/g, ' ')
        .trim();
}

function resolvePubType(rawType) {
    const typeMap = {
        inproceedings: 'conference',
        conference: 'conference',
        article: 'journal',
        book: 'book',
        booklet: 'book',
        incollection: 'book',
        techreport: 'techreport',
        phdthesis: 'thesis',
        mastersthesis: 'thesis',
        misc: 'preprint',
        unpublished: 'preprint'
    };
    return typeMap[rawType] || 'misc';
}

function resolveEffectiveUrl(fields) {
    let pdfUrl = fields.url || null;

    if (fields.note) {
        const urlMatch = fields.note.match(/https?:\/\/[^\s\}]+/i);
        if (urlMatch) {
            pdfUrl = urlMatch[0];
        }
    }

    let finalUrl = fields.url || null;
    if (finalUrl) {
        finalUrl = finalUrl.replace(/\\url\{([^}]*)\}/g, '$1').replace(/[\{\}]/g, '');
    }

    if (fields.note) {
        const urlMatch = fields.note.match(/https?:\/\/[^\s\}]+/i);
        if (urlMatch && (!finalUrl || urlMatch[0].toLowerCase().includes('.pdf'))) {
            pdfUrl = urlMatch[0];
        }
    }

    if (pdfUrl) {
        pdfUrl = pdfUrl.replace(/\\url\{([^}]*)\}/g, '$1').replace(/[\{\}]/g, '').trim();
    }

    return pdfUrl || finalUrl;
}

function normalizeEntry(entry, stringDefs) {
    const fields = entry.fields;
    const rawType = entry.type.toLowerCase();

    let venue = fields.booktitle || fields.journal || '';
    const venueKey = venue.toLowerCase();
    if (stringDefs[venueKey]) {
        venue = stringDefs[venueKey];
    }
    venue = venue.replace(/#\s*"-?/g, ' ').replace(/-?"/g, '').trim();

    const typePriority = {
        book: 0,
        conference: 1,
        journal: 2,
        techreport: 3,
        thesis: 4,
        preprint: 5,
        misc: 6
    };

    const pubType = resolvePubType(rawType);
    const effectiveUrl = resolveEffectiveUrl(fields);
    const canonicalKey = fields.crossref || entry.key;

    return {
        key: entry.key,
        canonicalKey: canonicalKey,
        type: pubType,
        typePriority: typePriority[pubType] || 99,
        title: fields.title || 'Untitled',
        authors: formatAuthors(fields.author || ''),
        year: parseInt(fields.year) || 0,
        venue: cleanLatex(venue),
        pages: fields.pages || '',
        doi: fields.doi || null,
        url: effectiveUrl,
        eprint: fields.eprint || null,
        note: fields.note ? cleanLatex(fields.note) : null,
        awards: fields.note_award ? fields.note_award.split(';').map(a => cleanLatex(a.trim())).filter(Boolean) : [],
        keywords: fields.keywords ? fields.keywords.split(',').map(k => cleanLatex(k.trim())).filter(Boolean) : [],
        publisher: fields.publisher || null,
        volume: fields.volume || null,
        number: fields.number || null,
        originalIndex: entry.index,
        raw: entry.raw
    };
}

function formatAuthors(authorString) {
    if (!authorString) return '';

    const authors = authorString.split(/\s+and\s+/i);

    return authors.map(author => {
        author = author.replace(/\$\^[^$]*\$/g, '').trim();

        if (author.includes(',')) {
            const parts = author.split(',').map(p => p.trim());
            if (parts.length >= 2) return `${parts[1]} ${parts[0]}`;
        }

        return author;
    }).join(', ');
}

function httpGetJson(url) {
    return new Promise((resolve, reject) => {
        const client = url.startsWith('https') ? https : http;
        client.get(url, {
            headers: {
                'User-Agent': 'PublicationsArchiver/1.0 (mailto:publications-archive@example.com)'
            }
        }, (res) => {
            let data = '';
            res.on('data', (chunk) => data += chunk);
            res.on('end', () => {
                if (res.statusCode !== 200) {
                    return reject(new Error(`HTTP status ${res.statusCode}`));
                }
                try {
                    resolve(JSON.parse(data));
                } catch (e) {
                    reject(e);
                }
            });
        }).on('error', reject);
    });
}

function downloadFile(url, destPath, maxRedirects = 5) {
    return new Promise((resolve, reject) => {
        if (maxRedirects < 0) {
            return reject(new Error('Too many redirects'));
        }

        const client = url.startsWith('https') ? https : http;
        const request = client.get(url, {
            headers: {
                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
            }
        }, (response) => {
            if (response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
                const redirectUrl = new URL(response.headers.location, url).href;
                return resolve(downloadFile(redirectUrl, destPath, maxRedirects - 1));
            }

            if (response.statusCode !== 200) {
                return reject(new Error(`HTTP status ${response.statusCode}`));
            }

            const dir = path.dirname(destPath);
            if (!fs.existsSync(dir)) {
                fs.mkdirSync(dir, { recursive: true });
            }

            const fileStream = fs.createWriteStream(destPath);
            response.pipe(fileStream);

            fileStream.on('finish', () => {
                fileStream.close();
                resolve();
            });

            fileStream.on('error', (err) => {
                fs.unlink(destPath, () => {});
                reject(err);
            });
        });

        request.on('error', (err) => {
            reject(err);
        });
    });
}

async function lookupWorkByDoi(doi) {
    const cleanDoi = doi.replace(/https?:\/\/doi\.org\//, '').trim();
    const url = `https://api.openalex.org/works/https://doi.org/${cleanDoi}`;
    try {
        return await httpGetJson(url);
    } catch (err) {
        const queryUrl = `https://api.openalex.org/works?filter=doi:${encodeURIComponent(cleanDoi)}`;
        const results = await httpGetJson(queryUrl);
        return results && results.results && results.results.length > 0 ? results.results[0] : null;
    }
}

async function lookupWorkByTitle(title) {
    const queryUrl = `https://api.openalex.org/works?filter=title.search:${encodeURIComponent(title)}`;
    const results = await httpGetJson(queryUrl);
    if (!results || !results.results || results.results.length === 0) {
        return null;
    }
    const sTitle = title.toLowerCase().replace(/[^a-z0-9]/g, '');
    const match = results.results.find(w => {
        const wTitle = (w.title || '').toLowerCase().replace(/[^a-z0-9]/g, '');
        return wTitle === sTitle || wTitle.includes(sTitle) || sTitle.includes(wTitle);
    });
    return match || results.results[0];
}

function extractPdfFromWork(work) {
    if (!work) return null;
    if (work.best_oa_location && work.best_oa_location.pdf_url) {
        return work.best_oa_location.pdf_url;
    }
    if (work.primary_location && work.primary_location.pdf_url) {
        return work.primary_location.pdf_url;
    }
    if (work.locations) {
        for (const loc of work.locations) {
            if (loc.pdf_url) return loc.pdf_url;
        }
    }
    return null;
}

async function findPdfUrlFromOpenAlex(title, doi) {
    try {
        let work = null;
        if (doi) {
            work = await lookupWorkByDoi(doi);
        }
        if (!work && title) {
            work = await lookupWorkByTitle(title);
        }
        return extractPdfFromWork(work);
    } catch (e) {
    }
    return null;
}

async function findPdfUrlFromSemanticScholar(title) {
    try {
        const url = `https://api.semanticscholar.org/graph/v1/paper/search?query=${encodeURIComponent(title)}&limit=1&fields=title,openAccessPdf`;
        const res = await httpGetJson(url);
        return res && res.data && res.data.length > 0 && res.data[0].openAccessPdf
            ? res.data[0].openAccessPdf.url
            : null;
    } catch (e) {
    }
    return null;
}

async function resolvePublicationPdf(pub) {
    let pdfUrl = pub.url;

    if (!pdfUrl && pub.eprint) {
        const cleanEprint = pub.eprint.replace(/[\{\}]/g, '').trim();
        pdfUrl = `https://arxiv.org/pdf/${cleanEprint}.pdf`;
        console.log(`  -> Resolved via arXiv: ${pdfUrl}`);
        return pdfUrl;
    }

    if (!pdfUrl) {
        console.log(`  -> Fetching PDF URL via OpenAlex...`);
        pdfUrl = await findPdfUrlFromOpenAlex(pub.title, pub.doi);
    }

    if (!pdfUrl) {
        console.log(`  -> Fetching PDF URL via Semantic Scholar...`);
        pdfUrl = await findPdfUrlFromSemanticScholar(pub.title);
    }

    return pdfUrl;
}

async function processSinglePublication(pub, stats) {
    const yearFolder = path.join(PUB_DIR, String(pub.year || 'Unknown'));
    const fileName = `${pub.canonicalKey}.pdf`;
    const localDestPath = path.join(yearFolder, fileName);
    const relPathForReadme = `./publications/${pub.year || 'Unknown'}/${fileName}`;

    if (fs.existsSync(localDestPath)) {
        console.log(`  -> Already downloaded: ${relPathForReadme}`);
        stats.skippedExisting++;
        return relPathForReadme;
    }

    const pdfUrl = await resolvePublicationPdf(pub);

    if (pdfUrl) {
        console.log(`  -> Attempting download from: ${pdfUrl}`);
        try {
            await downloadFile(pdfUrl, localDestPath);
            console.log(`  ✨ Success! Downloaded to: ${relPathForReadme}`);
            stats.downloaded++;
            return relPathForReadme;
        } catch (err) {
            console.error(`  ❌ Failed to download from ${pdfUrl}: ${err.message}`);
            stats.failed++;
        }
    } else {
        console.log(`  ⚠️ No PDF URL found on BibTeX, arXiv, OpenAlex, or Semantic Scholar.`);
        stats.missing++;
    }

    return null;
}

async function main() {
    console.log('📚 Starting publications archiving pipeline...\n');

    if (!fs.existsSync(BIB_FILE)) {
        console.error(`Error: Bibliography file not found at ${BIB_FILE}`);
        process.exit(1);
    }

    const bibContent = fs.readFileSync(BIB_FILE, 'utf-8');
    const publications = parseBibTeX(bibContent);

    console.log(`Parsed ${publications.length} publication entries from bibliography.\n`);

    const stats = {
        total: publications.length,
        downloaded: 0,
        failed: 0,
        missing: 0,
        skippedExisting: 0
    };

    publications.sort((a, b) => b.year - a.year);

    const updatedPublications = [];

    for (let i = 0; i < publications.length; i++) {
        const pub = publications[i];
        const displayIndex = i + 1;
        console.log(`[${displayIndex}/${publications.length}] Processing "${pub.title}" (${pub.year})`);

        const localPdfPath = await processSinglePublication(pub, stats);
        updatedPublications.push({ ...pub, localPdfPath });
    }

    console.log('\nGenerating index README.md...');
    generateReadme(updatedPublications);
    console.log('✨ README.md generated successfully.');

    console.log('\n==========================================');
    console.log('Archiving Pipeline Run Summary');
    console.log('==========================================');
    console.log(`Total publications parsed:  ${stats.total}`);
    console.log(`Downloaded in this run:     ${stats.downloaded}`);
    console.log(`Skipped (already exists):   ${stats.skippedExisting}`);
    console.log(`Failed to download:         ${stats.failed}`);
    console.log(`No PDF URL found anywhere:  ${stats.missing}`);
    console.log('==========================================\n');
}

function formatStatLine(counts, key, name) {
    return counts[key] ? `- **${name}:** ${counts[key]}\n` : '';
}

function formatPubDetails(pub) {
    let details = `*${pub.authors}*\n`;
    details += `*${pub.venue}*, ${pub.year}.\n`;

    const links = [];
    if (pub.localPdfPath) {
        links.push(`[📄 PDF](${pub.localPdfPath})`);
    } else if (pub.url) {
        links.push(`[🌐 External URL](${pub.url})`);
    }
    if (pub.doi) {
        links.push(`[🔗 DOI: ${pub.doi}](https://doi.org/${pub.doi})`);
    }

    if (links.length > 0) {
        details += `${links.join(' | ')}\n`;
    }

    const bibtexStr = pub.raw || `@misc{${pub.canonicalKey},\n  title = {${pub.title}},\n  author = {${pub.authors}},\n  year = {${pub.year}}\n}`;
    details += `\n<details>\n<summary>Cite (BibTeX)</summary>\n\n\`\`\`bibtex\n${bibtexStr}\n\`\`\`\n</details>\n\n`;

    return details;
}

function generateReadme(publications) {
    let markdown = `# Academic Publications Archive\n\n`;
    markdown += `This repository contains a self-hosted local archive of academic papers. The canonical bibliography list is managed in [cv.bib](./cv.bib).\n\n`;

    const counts = {};
    publications.forEach(p => {
        counts[p.type] = (counts[p.type] || 0) + 1;
    });

    markdown += `### Archive Statistics\n`;
    markdown += `- **Total Publications:** ${publications.length}\n`;
    markdown += formatStatLine(counts, 'conference', 'Conference Papers');
    markdown += formatStatLine(counts, 'journal', 'Journal Articles');
    markdown += formatStatLine(counts, 'preprint', 'Preprints / Posters');
    markdown += formatStatLine(counts, 'book', 'Books & Book Chapters');
    markdown += formatStatLine(counts, 'thesis', 'Theses');
    markdown += formatStatLine(counts, 'techreport', 'Technical Reports');
    markdown += `\n---\n\n`;

    const groupedByYear = {};
    publications.forEach(pub => {
        const year = pub.year || 'Unknown';
        if (!groupedByYear[year]) groupedByYear[year] = [];
        groupedByYear[year].push(pub);
    });

    const sortedYears = Object.keys(groupedByYear).sort((a, b) => b - a);

    sortedYears.forEach(year => {
        markdown += `## ${year}\n\n`;

        const pubsInYear = groupedByYear[year].sort((a, b) => a.typePriority - b.typePriority);

        pubsInYear.forEach((pub, index) => {
            const pubNum = index + 1;
            markdown += `**[${pubNum}] ${pub.title}**\n`;
            markdown += formatPubDetails(pub);
        });

        markdown += `\n`;
    });

    fs.writeFileSync(README_FILE, markdown, 'utf-8');
}

main().catch(console.error);