Added some tools and started working on automation

.github/workflows/automated_tool_search.yml

@@ -19,41 +19,66 @@ jobs:
           python-version: '3.10'
 
       - name: Install Python Dependencies
-        run: pip install google-genai
+        run: pip install google-genai flask
 
       - name: Run Tool Search
         env:
           GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
         run: python search_tools.py
 
+      - name: Check for new submissions
+        id: check_subs
+        run: |
+          count=$(wc -l < new_submissions.csv)
+          if [ "$count" -gt 1 ]; then
+            echo "has_new=true" >> $GITHUB_OUTPUT
+          else
+            echo "has_new=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: Setup R
+        if: steps.check_subs.outputs.has_new == 'true'
         uses: r-lib/actions/setup-r@v2
         with:
           r-version: 'release'
 
       - name: Install R Dependencies
+        if: steps.check_subs.outputs.has_new == 'true'
         run: |
-          install.packages(c("readr", "jsonlite", "docopt", "pbapply", "magrittr", "futile.logger", "aRxiv", "anytime", "citecorp", "stringr", "tidyverse", "gh", "here"))
+          install.packages(c("readr", "jsonlite", "docopt", "pbapply", "magrittr", "futile.logger", "aRxiv", "anytime", "citecorp", "stringr", "tidyverse", "gh", "here", "rcrossref", "xml2", "rvest", "lubridate", "scales", "viridis", "plotly", "htmlwidgets", "reshape2", "cowplot", "widgetframe", "fs", "plyr", "progress"))
         shell: Rscript {0}
 
       - name: Run Submission Check
+        if: steps.check_subs.outputs.has_new == 'true'
         env:
           GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
         run: Rscript -e "source('new_submission_check.R'); check_new_submission()"
 
+      - name: Update Database Build
+        if: steps.check_subs.outputs.has_new == 'true'
+        run: Rscript build_lrs_db.R --no-packages
+
+      - name: Test Website Server
+        if: steps.check_subs.outputs.has_new == 'true'
+        run: |
+          python docs/test_server.py &
+          sleep 5
+          curl -f http://127.0.0.1:5000/
+          kill %1
+
+      - name: Generate PR Message
+        if: steps.check_subs.outputs.has_new == 'true'
+        env:
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+        run: python generate_pr_message.py
+
       - name: Create Pull Request
+        if: steps.check_subs.outputs.has_new == 'true'
         uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: "chore: weekly automated tool search"
+          commit-message: "chore: weekly automated tool search update"
           title: "Automated Weekly Tool Search Results"
-          body: |
-            This PR contains potential new tools found during the weekly automated search.
-
-            Please review the generated files:
-            - `new_submissions_no_conflicts.csv`
-            - `new_submissions_conflicts.csv`
-
-            Merge these manually into `long_read_tools_master.csv` if they are valid.
+          body-path: pr_message.txt
           branch: automated-tool-search
           delete-branch: true

.gitignore

@@ -3,5 +3,3 @@
 .RData
 .Ruserdata
 *.log
-.Rhistory
-build_lrs_db.R

append_script.py

@@ -0,0 +1,36 @@
+import pandas as pd
+
+# Load master
+try:
+    master = pd.read_csv('long_read_tools_master.csv', encoding='latin-1')
+except Exception as e:
+    master = pd.read_csv('long_read_tools_master.csv', encoding='cp1252')
+
+# Load no conflicts
+try:
+    no_conflicts = pd.read_csv('new_submissions_no_conflicts.csv', encoding='utf-8')
+    no_conflicts = no_conflicts.rename(columns={'Name': 'Tool', 'DOIs': 'DOI', 'Platform': 'Programming_Language', 'Description': 'Details', 'Code': 'Source'})
+    master = pd.concat([master, no_conflicts], ignore_index=True)
+except Exception as e:
+    pass
+
+# Load conflicts and resolve
+try:
+    conflicts = pd.read_csv('new_submissions_conflicts.csv', encoding='utf-8')
+    conflicts = conflicts.rename(columns={'Name': 'Tool', 'DOIs': 'DOI', 'Platform': 'Programming_Language', 'Description': 'Details', 'Code': 'Source'})
+
+    # Resolve conflicts by preferring auto-detected platform and license
+    conflicts['Programming_Language'] = conflicts['Platform_auto'].fillna(conflicts['Programming_Language'])
+    conflicts['License'] = conflicts['License_auto'].fillna(conflicts['License'])
+
+    # Drop the extra columns added during check
+    cols_to_drop = ['Duplicated', 'Platform_auto', 'License_auto', 'Github']
+    conflicts = conflicts.drop(columns=[c for c in cols_to_drop if c in conflicts.columns])
+
+    master = pd.concat([master, conflicts], ignore_index=True)
+except Exception as e:
+    pass
+
+# Save master
+master.to_csv('long_read_tools_master.csv', index=False, encoding='latin-1')
+print("Successfully appended tools to master CSV.")

build_lrs_db.R

@@ -0,0 +1,182 @@
+#!/usr/bin/env Rscript
+
+"Usage: process_csv [options]
+
+This utility script converts the 'long_read_tools_master.csv' spreadsheet to a set
+of files in 'docs/data' required for the long-read-tools.org website.
+
+Options:
+-s --no-shields    Skip downloading of shields
+-c --no-citations  Skip getting Crossref citations
+-p --no-packages   Skip downloading of package lists
+-a --no-analysis   Skip plotting for analysis page
+" -> doc
+
+#### PACKAGES ####
+
+suppressPackageStartupMessages({
+    library(readr)
+    library(jsonlite)
+    library(docopt)
+    library(pbapply)
+    library(magrittr)
+    library(futile.logger)
+    library(aRxiv)
+    library(anytime)
+    library(citecorp)
+    library(stringr)
+    library(tidyverse)
+    library(gh)
+    library(here)
+})
+
+#### SOURCE ####
+
+source("R/load.R")
+source("R/manipulation.R")
+source("R/misc.R")
+source("R/output.R")
+source("R/plotting.R")
+source("R/references.R")
+source("R/repositories.R")
+source("R/titles.R")
+
+#### FUNCTIONS ####
+
+#' Process CSV
+#'
+#' Process `long_read_tools_master.csv` and create the various output files
+process_csv <- function(skip_shields = FALSE, skip_cites = FALSE,
+                        skip_packages = FALSE, skip_analysis = FALSE) {
+
+    flog.info("***** START DATABASE GENERATION *****")
+
+    flog.info("***** Loading data ******")
+    # Load data
+    swsheet <- get_swsheet()
+
+    if (!skip_packages) {
+        pkgs <- get_pkgs()
+    } else {
+        msg <- "Skipping downloading of package lists, they may be out of date!"
+        flog.warn(msg)
+        pkgs <- list(BioC  = "DUMMY",
+                     CRAN  = "DUMMY",
+                     PyPI  = "DUMMY",
+                     Conda = "DUMMY")
+    }
+    descs <- get_descriptions()
+    titles_cache <- get_cached_titles()
+
+    # Add new titles
+     titles_cache <- add_to_titles_cache(swsheet, titles_cache)
+
+    # Check repositories
+    repos <- check_repos(swsheet$Name, pkgs)
+
+    # Process table
+    flog.info("***** Processing table ******")
+    swsheet <- swsheet %>%
+        add_refs(titles_cache, skip_cites) %>%
+        add_github() %>%
+        add_repos(repos)
+
+    # Convert to tidy format for technologies
+    tidysw_tech <- tidy_swsheet_tech(swsheet)
+
+    # Add technologies to table
+    swsheet <- add_techs(swsheet, tidysw_tech)
+
+    # Convert to tidy format for categories
+    tidysw_cat <- tidy_swsheet_cat(swsheet)
+
+    # Add categories to table
+    swsheet <- add_cats(swsheet, tidysw_cat)
+
+    flog.info("***** Downloading shields ******")
+    # Get shields
+    if (!skip_shields) {
+        get_shields(swsheet)
+    } else {
+        msg <- paste("Downloading of shields has been skipped,",
+                     "they may be out of date!")
+        flog.warn(msg)
+    }
+
+    # Convert to JSON
+    flog.info("***** Converting to JSON *****")
+    flog.info("Converting table...")
+    make_table_json(swsheet)
+    make_tools_json(tidysw_cat)
+    make_cats_json(tidysw_cat, swsheet, descs)
+    make_techs_json(tidysw_tech, swsheet, descs)
+
+    # generate benchmark tab JSON
+    make_benchmark_json()
+
+    # geerating the quick-start workflow JSON
+    make_qs_json()
+
+    flog.info("***** Performing analysis *****")
+    if (!skip_analysis) {
+        # Make plots
+        flog.info("Plotting published tools over time...")
+        plot_number(swsheet)
+        flog.info("Plotting publication status...")
+        plot_publication(swsheet)
+        flog.info("Plotting platforms...")
+        plot_platforms(swsheet)
+        flog.info("Plotting technologies...")
+        plot_technologies(swsheet)
+        flog.info("Plotting categories...")
+        plot_categories(swsheet)
+    } else {
+        flog.warn("Skipping analysis, this may be out of date!")
+    }
+
+    if (sum(c(skip_shields, skip_cites, skip_packages, skip_analysis)) == 0) {
+        write_footer()
+        flog.info("****** All processing complete *****")
+    } else {
+        msg <- paste("Some processing has been skipped, last updated time will",
+                     "not be set!")
+        flog.warn(msg)
+    }
+
+    flog.info("**** PROCESSING FINISHED *****")
+}
+
+#### MAIN CODE ####
+
+logo <- "
+
+
+
+            LL   RRRR  TTTTTT    OOO  RRRR   GGGG                
+            LL   RR  R   TT     O   O RR  R G
+            LL   RR R    TT  OO O   O RR R  G  GGG
+            LLLL RR  R   TT      OOO  RR  R  GGGGG
+
+
+
+"
+
+cat(logo)
+
+# Show warnings
+options(warn = 1)
+
+# Setup logging
+layout <- layout.format('[~t] ~l: ~m')
+invisible(flog.layout(layout))
+invisible(flog.appender(appender.tee("processing.log")))
+
+# Setup progress bar
+pboptions(type = "timer", char = "=", style = 3)
+
+# Get options
+opts <- docopt(doc)
+
+# Process table
+process_csv(opts[["--no-shields"]], opts[["--no-citations"]],
+            opts[["--no-packages"]], opts[["--no-analysis"]])