Asap 215 document audit export

app/api/asap_pdf/api.rb

@@ -6,44 +6,60 @@ class API < Grape::API::Instance
     format :json
 
     http_basic do |email, password|
-      user = User.find_by(email: email)
-      user&.valid_password?(password)
+      @user = User.find_by(email: email)
+      @user&.valid_password?(password)
     end
 
     rescue_from ActiveRecord::RecordNotFound do |e|
       error!({error: e.message}, 404)
     end
 
     desc "Return list of sites" do
-      detail "Returns a list of all sites in the system"
+      detail "Returns a list of all sites the user has access to."
       tags ["Sites"]
       produces ["application/json"]
       failure [[401, "Unauthorized"], [403, "Forbidden"]]
+      security [{basic_auth: []}]
     end
     get "/sites" do
-      Site.all
+      if @user.is_site_admin
+        Site.all
+      else
+        @user.site.present? ? [@user.site] : []
+      end
     end
 
-    desc "Return a specific site" do
-      detail "Returns detailed information about a specific site"
-      tags ["Sites"]
+    desc "List documents related to site." do
+      detail "A paginated list of documents related to a site."
+      tags ["Documents"]
       produces ["application/json"]
+      consumes ["application/json"]
       failure [
+        [400, "Bad Request - Invalid parameters"],
         [401, "Unauthorized"],
         [403, "Forbidden"],
         [404, "Site not found"]
       ]
+      named "List documents"
     end
     params do
       requires :id, type: Integer, desc: "Site ID"
+      optional :page, type: Integer, desc: "Page number for pagination", default: 0
+      optional :items_per_page, type: Integer, desc: "Items per page", default: 25
     end
-    get "/sites/:id" do
-      Site.find(params[:id])
+    get "/sites/:id/documents" do
+      items_per_page = params[:items_per_page].nil? ? 25 : params[:items_per_page].to_i
+      page = params[:page].nil? ? 0 : params[:page].to_i
+      unless @user.is_site_admin || params[:id] == @user.site_id
+        error!("Unauthorized", 401)
+      end
+      site = Site.find(params[:id])
+      {documents: site.documents.limit(items_per_page).offset(page * items_per_page).order(id: :asc)}
     end
 
-    desc "Discover documents for a site" do
-      detail "Creates or updates documents for a specific site based on the provided URLs and timestamps"
-      tags ["Documents"]
+    desc "List document inferences for a document." do
+      detail "List LLM document inferences (summary or exception check) for a document."
+      tags ["Document Inferences"]
       produces ["application/json"]
       consumes ["application/json"]
       failure [
@@ -52,28 +68,16 @@ class API < Grape::API::Instance
         [403, "Forbidden"],
         [404, "Site not found"]
       ]
-      named "Create Documents"
     end
     params do
-      requires :id, type: Integer, desc: "Site ID"
-      requires :documents, type: Array do
-        requires :url, type: String, desc: "Document URL"
-        requires :modification_date, type: DateTime, desc: "Document's last modified timestamp"
-      end
+      requires :id, type: Integer, desc: "Document ID"
     end
-    post "/sites/:id/documents" do
-      site = Site.find(params[:id])
-      documents = site.discover_documents!(params[:documents], true)
-
-      status 201
-      {documents: documents.map { |doc|
-        {
-          id: doc.id,
-          url: doc.url,
-          document_status: doc.document_status,
-          s3_path: doc.s3_path
-        }
-      }}
+    get "/documents/:id/document_inference" do
+      document = Document.find(params[:id])
+      unless @user.is_site_admin || document.site_id == @user.site_id
+        error!("Unauthorized", 401)
+      end
+      {document_inferences: document.document_inferences.order(id: :asc)}
     end
 
     desc "Adds or updates a document inference with type" do
@@ -91,12 +95,20 @@ class API < Grape::API::Instance
     params do
       requires :id, type: Integer, desc: "Document ID"
       requires :inference_type, type: String, desc: "Document inference type", values: ["summary", "exception"]
-      requires :result, type: Hash, desc: "Value of document inference"
+      optional :result, type: Hash, desc: "Value of document inference" do
+        optional "summary", type: String, desc: "For inference_type summary, generated summary"
+        optional "is_<exception type>", type: String, desc: "For inference_type exception, the exception type"
+        optional "why_<exception type>", type: String, desc: "For inference_type exception, the LLM explanation"
+      end
     end
     post "/documents/:id/inference" do
       status 201
+      document = Document.find(params[:id])
+      unless @user.is_site_admin || document.site_id == @user.site_id
+        error!("Unauthorized", 401)
+      end
       if params[:inference_type] == "summary"
-        inference = DocumentInference.create(document_id: params[:id], inference_type: "summary")
+        inference = document.document_inferences.new(inference_type: "summary")
         inference.inference_value = params[:result]["summary"]
         inference.is_active = true
         inference.inference_model_name = params[:result]["inference_model"]
@@ -107,7 +119,7 @@ class API < Grape::API::Instance
         ["individualized", "archival", "application", "third_party"].each do |type|
           result_boolean = "is_#{type}"
           unless params[:result][result_boolean].nil?
-            inference = DocumentInference.create(document_id: params[:id], inference_type: "exception:#{result_boolean}")
+            inference = document.document_inferences.new(inference_type: "exception:#{result_boolean}")
             inference.inference_value = params[:result][result_boolean] ? "True" : "False"
             inference.inference_confidence = params[:result]["#{result_boolean}_confidence"]
             inference.inference_reason = params[:result]["why_#{type}"]
@@ -121,41 +133,19 @@ class API < Grape::API::Instance
     end
 
     add_swagger_documentation(
+      doc_version: "1.0.0",
       mount_path: "/swagger_doc",
-      openapi_version: "3.0.1",
       info: {
         title: "ASAP PDF API",
-        description: "API for managing ASAP PDF resources and document processing",
+        description: "API for managing ASAP PDF resources and document processing. <strong>Note: Basic authentication is required for all endpoints.</strong>",
         version: "1.0.0"
       },
       tags: [
-        {name: "Sites", description: "Site management operations"},
-        {name: "Documents", description: "Document processing operations"}
+        {name: "Sites", description: "Site operations"},
+        {name: "Documents", description: "Document operations"},
+        {name: "Document Inferences", description: "Document Inference operations"}
       ],
-      components: {
-        schemas: {
-          Site: {
-            type: "object",
-            properties: {
-              id: {type: "integer", description: "Site ID"},
-              name: {type: "string", description: "Site name"},
-              location: {type: "string", description: "Site location"},
-              primary_url: {type: "string", description: "Primary URL of the site"}
-            },
-            required: ["id", "name", "location", "primary_url"]
-          },
-          Document: {
-            type: "object",
-            properties: {
-              id: {type: "integer", description: "Document ID"},
-              url: {type: "string", description: "Document URL"},
-              document_status: {type: "string", description: "Current status of the document"},
-              s3_path: {type: "string", description: "S3 storage path"}
-            },
-            required: ["id", "url", "document_status", "s3_path"]
-          }
-        }
-      }
+      models: []
     )
   end
 end

app/controllers/configurations_controller.rb

@@ -3,43 +3,35 @@ class ConfigurationsController < AuthenticatedController
 
   before_action :ensure_user_site_admin
 
-  # This form is only for local development.
-  # Python components expect to use staging keys for local development.
-  ASAP_API_USER = "asap-pdf/staging/RAILS_API_USER"
-  ASAP_API_PASSWORD = "asap-pdf/staging/RAILS_API_PASSWORD"
-  GOOGLE_API_SECRET_NAME = "asap-pdf/staging/GOOGLE_AI_KEY"
-  ANTHROPIC_API_SECRET_NAME = "asap-pdf/staging/ANTHROPIC_KEY"
-  GOOGLE_EVAL_SERVICE_ACCOUNT_CREDS = "asap-pdf/staging/GOOGLE_SERVICE_ACCOUNT"
-  GOOGLE_EVAL_SHEET_ID = "asap-pdf/staging/GOOGLE_SHEET_ID_EVALUATION"
-
   def initialize
     super
     @secret_manager = AwsLocalSecretManager.new
+    @secret_names = Rails.configuration.local_secret_names
   end
 
   def edit
     @config = {
       localstack_not_reachable: false
     }
-    response = @secret_manager.get_secret!(GOOGLE_API_SECRET_NAME)
+    response = @secret_manager.get_secret!(@secret_names[:google_api])
     @config["google_ai_api_key"] = response.secret_string if response.present?
-    response = @secret_manager.get_secret!(ANTHROPIC_API_SECRET_NAME)
+    response = @secret_manager.get_secret!(@secret_names[:anthropic_api])
     @config["anthropic_api_key"] = response.secret_string if response.present?
-    response = @secret_manager.get_secret!(GOOGLE_EVAL_SERVICE_ACCOUNT_CREDS)
+    response = @secret_manager.get_secret!(@secret_names[:google_eval_service_account])
     @config["google_evaluation_service_account_credentials"] = response.secret_string if response.present?
-    response = @secret_manager.get_secret!(GOOGLE_EVAL_SHEET_ID)
+    response = @secret_manager.get_secret!(@secret_names[:google_eval_sheet_id])
     @config["google_evaluation_sheet_id"] = response.secret_string if response.present?
   rescue Seahorse::Client::NetworkingError
     @config["localstack_not_reachable"] = true
   end
 
   def update
-    @secret_manager.set_secret!(GOOGLE_API_SECRET_NAME, params[:config][:google_ai_api_key])
-    @secret_manager.set_secret!(ANTHROPIC_API_SECRET_NAME, params[:config][:anthropic_api_key])
-    @secret_manager.set_secret!(ASAP_API_USER, Rails.application.credentials.config[:api_user])
-    @secret_manager.set_secret!(ASAP_API_PASSWORD, Rails.application.credentials.config[:api_password])
-    @secret_manager.set_secret!(GOOGLE_EVAL_SERVICE_ACCOUNT_CREDS, params[:config][:google_evaluation_service_account_credentials])
-    @secret_manager.set_secret!(GOOGLE_EVAL_SHEET_ID, params[:config][:google_evaluation_sheet_id])
+    @secret_manager.set_secret!(@secret_names[:google_api], params.dig(:config, :google_ai_api_key))
+    @secret_manager.set_secret!(@secret_names[:anthropic_api], params.dig(:config, :anthropic_api_key))
+    @secret_manager.set_secret!(@secret_names[:asap_api_user], Rails.application.credentials.config[:api_user])
+    @secret_manager.set_secret!(@secret_names[:asap_api_password], Rails.application.credentials.config[:api_password])
+    @secret_manager.set_secret!(@secret_names[:google_eval_service_account], params.dig(:config, :google_evaluation_service_account_credentials))
+    @secret_manager.set_secret!(@secret_names[:google_eval_sheet_id], params.dig(:config, :google_evaluation_sheet_id))
     redirect_to edit_configuration_path, notice: "Configuration updated successfully. API user set to Rails config values."
   rescue => e
     redirect_to edit_configuration_path, alert: "Error updating configuration: #{e.message}"

app/controllers/documents_controller.rb

@@ -4,10 +4,10 @@ class DocumentsController < AuthenticatedController
 
   protect_from_forgery with: :exception
   skip_before_action :verify_authenticity_token, only: [:update_document_category, :update_accessibility_recommendation, :update_notes, :update_summary_inference, :update_recommendation_inference]
-  before_action :set_site, only: [:index, :modal_content, :batch_update]
-  before_action :set_document, except: [:index, :batch_update]
-  before_action :ensure_user_site_access, only: [:index, :modal_content, :batch_update]
-  before_action :ensure_user_document_access, except: [:index, :modal_content, :batch_update]
+  before_action :set_site, only: [:index, :insights, :audit_exports, :modal_content, :batch_update]
+  before_action :set_document, except: [:index, :insights, :audit_exports, :batch_update]
+  before_action :ensure_user_site_access, only: [:index, :insights, :audit_exports, :modal_content, :batch_update]
+  before_action :ensure_user_document_access, except: [:index, :insights, :audit_exports, :modal_content, :batch_update]
 
   def modal_content
     render partial: "modal_content", locals: {document: @document}
@@ -29,6 +29,110 @@ def index
     @filters_for_sorts = query_params [:sort, :direction, :page]
   end
 
+  def insights
+    # Build document list.
+    @documents = @site.documents
+      .by_category(params[:category])
+      .by_department(params[:department])
+    # Create binned date data for visualization.
+    # First, gather all documents by year
+    year_groups = @documents.group_by(&:modification_year).map { |label, year_documents| [label, year_documents.size] }
+    # Extract and remove "Unknown" to handle separately
+    unknown_group = year_groups.find { |item| item[0] == "Unknown" }
+    year_groups = year_groups.reject { |item| item[0] == "Unknown" }
+    year_groups = year_groups.select do |item|
+      Integer(item[0])
+      true
+    rescue
+      if unknown_group.nil?
+        unknown_group = ["Unknown", 0]
+      end
+      unknown_group[1] += 1
+      false
+    end
+    # Convert to integers for sorting and calculations
+    year_groups = year_groups.map { |year, count| [Integer(year), count] }
+    # Create bins based on specific year ranges
+    binned_data = []
+    bins = [
+      ["< 2000", -Float::INFINITY..1999],
+      ["2000-2005", 2000..2005],
+      ["2006-2011", 2006..2011],
+      ["2012-2017", 2012..2017],
+      ["2018-2023", 2018..2023],
+      ["> 2023", 2024..Float::INFINITY]
+    ]
+    bins.each do |label, range|
+      count = year_groups.filter_map { |year, count| count if range.cover?(year) }.sum
+      binned_data << [label, count]
+    end
+    # Add the "Unknown" group if it exists (placing it at the end)
+    binned_data << unknown_group if unknown_group
+    @document_years = binned_data
+    # Create table data.
+    default_group = Document::DECISION_TYPES.keys.map { |status| [status, 0] }.to_h
+    @category_groups = {}
+    @documents.group([:document_category, :accessibility_recommendation]).count.each do |groups, group_count|
+      @category_groups[groups[0]] = default_group.clone if @category_groups[groups[0]].nil?
+      if Document::DECISION_TYPES.keys.exclude? groups[1]
+        parent = Document::DECISION_TYPES.keys.find do |key|
+          if Document::DECISION_TYPES[key]["children"].present? && Document::DECISION_TYPES[key]["children"].key?(groups[1])
+            key
+          end
+        end
+        if parent.present?
+          groups[1] = parent
+        end
+      end
+      @category_groups[groups[0]][groups[1]] += group_count
+    end
+    @category_groups.each do |key, child_hash|
+      sum = child_hash.values.sum
+      child_hash["Total"] = sum
+    end
+    @category_groups = @category_groups.sort.to_h
+    # Work on document links.
+    @document_links = {
+      complexity: [
+        {title: Document::SIMPLE_STATUS, params: query_params.merge({complexity: Document::SIMPLE_STATUS})},
+        {title: Document::COMPLEX_STATUS, params: query_params.merge({complexity: Document::COMPLEX_STATUS})}
+      ],
+      years: bins.map do |label, range|
+        document_count = @document_years.find { |item| item[0] == label }
+        if document_count[1] == 0
+          next
+        end
+        start_date = (range.begin == -Float::INFINITY) ? nil : "#{range.begin}-01-01"
+        end_date = (range.end == Float::INFINITY) ? nil : "#{range.end}-12-31"
+        {
+          title: label,
+          params: query_params.merge(
+            start_date: start_date,
+            end_date: end_date
+          ).compact
+        }
+      end.compact,
+      decision: @documents.pluck(:accessibility_recommendation).uniq.map do |decision|
+        {
+          title: decision,
+          params: query_params.merge(
+            accessibility_recommendation: decision
+          )
+        }
+      end
+    }
+  end
+
+  def audit_exports
+    @export_links = []
+    @error_message = nil
+    begin
+      @export_links = @site.get_document_audit_link_hashes!
+    rescue => e
+      @error_message = e.message
+    end
+  end
+
   def serve_document_url
     response = HTTParty.get(@document.normalized_url)
     if response.success?