diff --git a/app/api/asap_pdf/api.rb b/app/api/asap_pdf/api.rb index 33befe17..03a8e66b 100644 --- a/app/api/asap_pdf/api.rb +++ b/app/api/asap_pdf/api.rb @@ -6,8 +6,8 @@ class API < Grape::API::Instance format :json http_basic do |email, password| - user = User.find_by(email: email) - user&.valid_password?(password) + @user = User.find_by(email: email) + @user&.valid_password?(password) end rescue_from ActiveRecord::RecordNotFound do |e| @@ -15,35 +15,51 @@ class API < Grape::API::Instance end desc "Return list of sites" do - detail "Returns a list of all sites in the system" + detail "Returns a list of all sites the user has access to." tags ["Sites"] produces ["application/json"] failure [[401, "Unauthorized"], [403, "Forbidden"]] + security [{basic_auth: []}] end get "/sites" do - Site.all + if @user.is_site_admin + Site.all + else + @user.site.present? ? [@user.site] : [] + end end - desc "Return a specific site" do - detail "Returns detailed information about a specific site" - tags ["Sites"] + desc "List documents related to site." do + detail "A paginated list of documents related to a site." + tags ["Documents"] produces ["application/json"] + consumes ["application/json"] failure [ + [400, "Bad Request - Invalid parameters"], [401, "Unauthorized"], [403, "Forbidden"], [404, "Site not found"] ] + named "List documents" end params do requires :id, type: Integer, desc: "Site ID" + optional :page, type: Integer, desc: "Page number for pagination", default: 0 + optional :items_per_page, type: Integer, desc: "Items per page", default: 25 end - get "/sites/:id" do - Site.find(params[:id]) + get "/sites/:id/documents" do + items_per_page = params[:items_per_page].nil? ? 25 : params[:items_per_page].to_i + page = params[:page].nil? ? 0 : params[:page].to_i + unless @user.is_site_admin || params[:id] == @user.site_id + error!("Unauthorized", 401) + end + site = Site.find(params[:id]) + {documents: site.documents.limit(items_per_page).offset(page * items_per_page).order(id: :asc)} end - desc "Discover documents for a site" do - detail "Creates or updates documents for a specific site based on the provided URLs and timestamps" - tags ["Documents"] + desc "List document inferences for a document." do + detail "List LLM document inferences (summary or exception check) for a document." + tags ["Document Inferences"] produces ["application/json"] consumes ["application/json"] failure [ @@ -52,28 +68,16 @@ class API < Grape::API::Instance [403, "Forbidden"], [404, "Site not found"] ] - named "Create Documents" end params do - requires :id, type: Integer, desc: "Site ID" - requires :documents, type: Array do - requires :url, type: String, desc: "Document URL" - requires :modification_date, type: DateTime, desc: "Document's last modified timestamp" - end + requires :id, type: Integer, desc: "Document ID" end - post "/sites/:id/documents" do - site = Site.find(params[:id]) - documents = site.discover_documents!(params[:documents], true) - - status 201 - {documents: documents.map { |doc| - { - id: doc.id, - url: doc.url, - document_status: doc.document_status, - s3_path: doc.s3_path - } - }} + get "/documents/:id/document_inference" do + document = Document.find(params[:id]) + unless @user.is_site_admin || document.site_id == @user.site_id + error!("Unauthorized", 401) + end + {document_inferences: document.document_inferences.order(id: :asc)} end desc "Adds or updates a document inference with type" do @@ -91,12 +95,20 @@ class API < Grape::API::Instance params do requires :id, type: Integer, desc: "Document ID" requires :inference_type, type: String, desc: "Document inference type", values: ["summary", "exception"] - requires :result, type: Hash, desc: "Value of document inference" + optional :result, type: Hash, desc: "Value of document inference" do + optional "summary", type: String, desc: "For inference_type summary, generated summary" + optional "is_", type: String, desc: "For inference_type exception, the exception type" + optional "why_", type: String, desc: "For inference_type exception, the LLM explanation" + end end post "/documents/:id/inference" do status 201 + document = Document.find(params[:id]) + unless @user.is_site_admin || document.site_id == @user.site_id + error!("Unauthorized", 401) + end if params[:inference_type] == "summary" - inference = DocumentInference.create(document_id: params[:id], inference_type: "summary") + inference = document.document_inferences.new(inference_type: "summary") inference.inference_value = params[:result]["summary"] inference.is_active = true inference.inference_model_name = params[:result]["inference_model"] @@ -107,7 +119,7 @@ class API < Grape::API::Instance ["individualized", "archival", "application", "third_party"].each do |type| result_boolean = "is_#{type}" unless params[:result][result_boolean].nil? - inference = DocumentInference.create(document_id: params[:id], inference_type: "exception:#{result_boolean}") + inference = document.document_inferences.new(inference_type: "exception:#{result_boolean}") inference.inference_value = params[:result][result_boolean] ? "True" : "False" inference.inference_confidence = params[:result]["#{result_boolean}_confidence"] inference.inference_reason = params[:result]["why_#{type}"] @@ -121,41 +133,19 @@ class API < Grape::API::Instance end add_swagger_documentation( + doc_version: "1.0.0", mount_path: "/swagger_doc", - openapi_version: "3.0.1", info: { title: "ASAP PDF API", - description: "API for managing ASAP PDF resources and document processing", + description: "API for managing ASAP PDF resources and document processing. Note: Basic authentication is required for all endpoints.", version: "1.0.0" }, tags: [ - {name: "Sites", description: "Site management operations"}, - {name: "Documents", description: "Document processing operations"} + {name: "Sites", description: "Site operations"}, + {name: "Documents", description: "Document operations"}, + {name: "Document Inferences", description: "Document Inference operations"} ], - components: { - schemas: { - Site: { - type: "object", - properties: { - id: {type: "integer", description: "Site ID"}, - name: {type: "string", description: "Site name"}, - location: {type: "string", description: "Site location"}, - primary_url: {type: "string", description: "Primary URL of the site"} - }, - required: ["id", "name", "location", "primary_url"] - }, - Document: { - type: "object", - properties: { - id: {type: "integer", description: "Document ID"}, - url: {type: "string", description: "Document URL"}, - document_status: {type: "string", description: "Current status of the document"}, - s3_path: {type: "string", description: "S3 storage path"} - }, - required: ["id", "url", "document_status", "s3_path"] - } - } - } + models: [] ) end end diff --git a/app/controllers/configurations_controller.rb b/app/controllers/configurations_controller.rb index b7b60453..0e4b11fa 100644 --- a/app/controllers/configurations_controller.rb +++ b/app/controllers/configurations_controller.rb @@ -3,43 +3,35 @@ class ConfigurationsController < AuthenticatedController before_action :ensure_user_site_admin - # This form is only for local development. - # Python components expect to use staging keys for local development. - ASAP_API_USER = "asap-pdf/staging/RAILS_API_USER" - ASAP_API_PASSWORD = "asap-pdf/staging/RAILS_API_PASSWORD" - GOOGLE_API_SECRET_NAME = "asap-pdf/staging/GOOGLE_AI_KEY" - ANTHROPIC_API_SECRET_NAME = "asap-pdf/staging/ANTHROPIC_KEY" - GOOGLE_EVAL_SERVICE_ACCOUNT_CREDS = "asap-pdf/staging/GOOGLE_SERVICE_ACCOUNT" - GOOGLE_EVAL_SHEET_ID = "asap-pdf/staging/GOOGLE_SHEET_ID_EVALUATION" - def initialize super @secret_manager = AwsLocalSecretManager.new + @secret_names = Rails.configuration.local_secret_names end def edit @config = { localstack_not_reachable: false } - response = @secret_manager.get_secret!(GOOGLE_API_SECRET_NAME) + response = @secret_manager.get_secret!(@secret_names[:google_api]) @config["google_ai_api_key"] = response.secret_string if response.present? - response = @secret_manager.get_secret!(ANTHROPIC_API_SECRET_NAME) + response = @secret_manager.get_secret!(@secret_names[:anthropic_api]) @config["anthropic_api_key"] = response.secret_string if response.present? - response = @secret_manager.get_secret!(GOOGLE_EVAL_SERVICE_ACCOUNT_CREDS) + response = @secret_manager.get_secret!(@secret_names[:google_eval_service_account]) @config["google_evaluation_service_account_credentials"] = response.secret_string if response.present? - response = @secret_manager.get_secret!(GOOGLE_EVAL_SHEET_ID) + response = @secret_manager.get_secret!(@secret_names[:google_eval_sheet_id]) @config["google_evaluation_sheet_id"] = response.secret_string if response.present? rescue Seahorse::Client::NetworkingError @config["localstack_not_reachable"] = true end def update - @secret_manager.set_secret!(GOOGLE_API_SECRET_NAME, params[:config][:google_ai_api_key]) - @secret_manager.set_secret!(ANTHROPIC_API_SECRET_NAME, params[:config][:anthropic_api_key]) - @secret_manager.set_secret!(ASAP_API_USER, Rails.application.credentials.config[:api_user]) - @secret_manager.set_secret!(ASAP_API_PASSWORD, Rails.application.credentials.config[:api_password]) - @secret_manager.set_secret!(GOOGLE_EVAL_SERVICE_ACCOUNT_CREDS, params[:config][:google_evaluation_service_account_credentials]) - @secret_manager.set_secret!(GOOGLE_EVAL_SHEET_ID, params[:config][:google_evaluation_sheet_id]) + @secret_manager.set_secret!(@secret_names[:google_api], params.dig(:config, :google_ai_api_key)) + @secret_manager.set_secret!(@secret_names[:anthropic_api], params.dig(:config, :anthropic_api_key)) + @secret_manager.set_secret!(@secret_names[:asap_api_user], Rails.application.credentials.config[:api_user]) + @secret_manager.set_secret!(@secret_names[:asap_api_password], Rails.application.credentials.config[:api_password]) + @secret_manager.set_secret!(@secret_names[:google_eval_service_account], params.dig(:config, :google_evaluation_service_account_credentials)) + @secret_manager.set_secret!(@secret_names[:google_eval_sheet_id], params.dig(:config, :google_evaluation_sheet_id)) redirect_to edit_configuration_path, notice: "Configuration updated successfully. API user set to Rails config values." rescue => e redirect_to edit_configuration_path, alert: "Error updating configuration: #{e.message}" diff --git a/app/controllers/documents_controller.rb b/app/controllers/documents_controller.rb index 6db1ea2a..67ba1251 100644 --- a/app/controllers/documents_controller.rb +++ b/app/controllers/documents_controller.rb @@ -4,10 +4,10 @@ class DocumentsController < AuthenticatedController protect_from_forgery with: :exception skip_before_action :verify_authenticity_token, only: [:update_document_category, :update_accessibility_recommendation, :update_notes, :update_summary_inference, :update_recommendation_inference] - before_action :set_site, only: [:index, :modal_content, :batch_update] - before_action :set_document, except: [:index, :batch_update] - before_action :ensure_user_site_access, only: [:index, :modal_content, :batch_update] - before_action :ensure_user_document_access, except: [:index, :modal_content, :batch_update] + before_action :set_site, only: [:index, :insights, :audit_exports, :modal_content, :batch_update] + before_action :set_document, except: [:index, :insights, :audit_exports, :batch_update] + before_action :ensure_user_site_access, only: [:index, :insights, :audit_exports, :modal_content, :batch_update] + before_action :ensure_user_document_access, except: [:index, :insights, :audit_exports, :modal_content, :batch_update] def modal_content render partial: "modal_content", locals: {document: @document} @@ -29,6 +29,110 @@ def index @filters_for_sorts = query_params [:sort, :direction, :page] end + def insights + # Build document list. + @documents = @site.documents + .by_category(params[:category]) + .by_department(params[:department]) + # Create binned date data for visualization. + # First, gather all documents by year + year_groups = @documents.group_by(&:modification_year).map { |label, year_documents| [label, year_documents.size] } + # Extract and remove "Unknown" to handle separately + unknown_group = year_groups.find { |item| item[0] == "Unknown" } + year_groups = year_groups.reject { |item| item[0] == "Unknown" } + year_groups = year_groups.select do |item| + Integer(item[0]) + true + rescue + if unknown_group.nil? + unknown_group = ["Unknown", 0] + end + unknown_group[1] += 1 + false + end + # Convert to integers for sorting and calculations + year_groups = year_groups.map { |year, count| [Integer(year), count] } + # Create bins based on specific year ranges + binned_data = [] + bins = [ + ["< 2000", -Float::INFINITY..1999], + ["2000-2005", 2000..2005], + ["2006-2011", 2006..2011], + ["2012-2017", 2012..2017], + ["2018-2023", 2018..2023], + ["> 2023", 2024..Float::INFINITY] + ] + bins.each do |label, range| + count = year_groups.filter_map { |year, count| count if range.cover?(year) }.sum + binned_data << [label, count] + end + # Add the "Unknown" group if it exists (placing it at the end) + binned_data << unknown_group if unknown_group + @document_years = binned_data + # Create table data. + default_group = Document::DECISION_TYPES.keys.map { |status| [status, 0] }.to_h + @category_groups = {} + @documents.group([:document_category, :accessibility_recommendation]).count.each do |groups, group_count| + @category_groups[groups[0]] = default_group.clone if @category_groups[groups[0]].nil? + if Document::DECISION_TYPES.keys.exclude? groups[1] + parent = Document::DECISION_TYPES.keys.find do |key| + if Document::DECISION_TYPES[key]["children"].present? && Document::DECISION_TYPES[key]["children"].key?(groups[1]) + key + end + end + if parent.present? + groups[1] = parent + end + end + @category_groups[groups[0]][groups[1]] += group_count + end + @category_groups.each do |key, child_hash| + sum = child_hash.values.sum + child_hash["Total"] = sum + end + @category_groups = @category_groups.sort.to_h + # Work on document links. + @document_links = { + complexity: [ + {title: Document::SIMPLE_STATUS, params: query_params.merge({complexity: Document::SIMPLE_STATUS})}, + {title: Document::COMPLEX_STATUS, params: query_params.merge({complexity: Document::COMPLEX_STATUS})} + ], + years: bins.map do |label, range| + document_count = @document_years.find { |item| item[0] == label } + if document_count[1] == 0 + next + end + start_date = (range.begin == -Float::INFINITY) ? nil : "#{range.begin}-01-01" + end_date = (range.end == Float::INFINITY) ? nil : "#{range.end}-12-31" + { + title: label, + params: query_params.merge( + start_date: start_date, + end_date: end_date + ).compact + } + end.compact, + decision: @documents.pluck(:accessibility_recommendation).uniq.map do |decision| + { + title: decision, + params: query_params.merge( + accessibility_recommendation: decision + ) + } + end + } + end + + def audit_exports + @export_links = [] + @error_message = nil + begin + @export_links = @site.get_document_audit_link_hashes! + rescue => e + @error_message = e.message + end + end + def serve_document_url response = HTTParty.get(@document.normalized_url) if response.success? diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index fde11e78..bc7aaa01 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -2,8 +2,8 @@ class SitesController < AuthenticatedController include Access include ParamsHelper - before_action :find_site, only: [:insights, :show, :edit, :update, :destroy] - before_action :ensure_user_site_access, only: [:insights, :show, :edit, :update, :destroy] + before_action :find_site, only: [:show, :edit, :update, :destroy, :create_workflow_audit_report] + before_action :ensure_user_site_access, only: [:show, :edit, :update, :destroy, :workflow_audit_report] def index @sites = if current_user.is_site_admin? @@ -13,100 +13,6 @@ def index end end - def insights - # Build document list. - @documents = @site.documents - .by_category(params[:category]) - .by_department(params[:department]) - # Create binned date data for visualization. - # First, gather all documents by year - year_groups = @documents.group_by(&:modification_year).map { |label, year_documents| [label, year_documents.size] } - # Extract and remove "Unknown" to handle separately - unknown_group = year_groups.find { |item| item[0] == "Unknown" } - year_groups = year_groups.reject { |item| item[0] == "Unknown" } - year_groups = year_groups.select do |item| - Integer(item[0]) - true - rescue - if unknown_group.nil? - unknown_group = ["Unknown", 0] - end - unknown_group[1] += 1 - false - end - # Convert to integers for sorting and calculations - year_groups = year_groups.map { |year, count| [Integer(year), count] } - # Create bins based on specific year ranges - binned_data = [] - bins = [ - ["< 2000", -Float::INFINITY..1999], - ["2000-2005", 2000..2005], - ["2006-2011", 2006..2011], - ["2012-2017", 2012..2017], - ["2018-2023", 2018..2023], - ["> 2023", 2024..Float::INFINITY] - ] - bins.each do |label, range| - count = year_groups.filter_map { |year, count| count if range.cover?(year) }.sum - binned_data << [label, count] - end - # Add the "Unknown" group if it exists (placing it at the end) - binned_data << unknown_group if unknown_group - @document_years = binned_data - # Create table data. - default_group = Document::DECISION_TYPES.keys.map { |status| [status, 0] }.to_h - @category_groups = {} - @documents.group([:document_category, :accessibility_recommendation]).count.each do |groups, group_count| - @category_groups[groups[0]] = default_group.clone if @category_groups[groups[0]].nil? - if Document::DECISION_TYPES.keys.exclude? groups[1] - parent = Document::DECISION_TYPES.keys.find do |key| - if Document::DECISION_TYPES[key]["children"].present? && Document::DECISION_TYPES[key]["children"].key?(groups[1]) - key - end - end - if parent.present? - groups[1] = parent - end - end - @category_groups[groups[0]][groups[1]] += group_count - end - @category_groups.each do |key, child_hash| - sum = child_hash.values.sum - child_hash["Total"] = sum - end - @category_groups = @category_groups.sort.to_h - # Work on document links. - @document_links = { - complexity: [ - {title: Document::SIMPLE_STATUS, params: query_params.merge({complexity: Document::SIMPLE_STATUS})}, - {title: Document::COMPLEX_STATUS, params: query_params.merge({complexity: Document::COMPLEX_STATUS})} - ], - years: bins.map do |label, range| - document_count = @document_years.find { |item| item[0] == label } - if document_count[1] == 0 - next - end - start_date = (range.begin == -Float::INFINITY) ? nil : "#{range.begin}-01-01" - end_date = (range.end == Float::INFINITY) ? nil : "#{range.end}-12-31" - { - title: label, - params: query_params.merge( - start_date: start_date, - end_date: end_date - ).compact - } - end.compact, - decision: @documents.pluck(:accessibility_recommendation).uniq.map do |decision| - { - title: decision, - params: query_params.merge( - accessibility_recommendation: decision - ) - } - end - } - end - def show @documents = @site.documents.order(created_at: :desc) end @@ -141,6 +47,36 @@ def destroy redirect_to sites_path, notice: "Site was successfully deleted.", status: :see_other end + def create_workflow_audit_report + export_links = [] + error_message = nil + begin + @site.export_document_audit!(current_user) + export_links = @site.get_document_audit_link_hashes! + rescue => e + error_message = e.message + end + render json: {html: render_to_string(partial: "documents/audit_export_list", formats: [:html], locals: {export_links: export_links, error: error_message})} + end + + def workflow_audit_report + s3_manager = AwsS3Manager.new + begin + key = params[:key] + key = key.start_with?("/") ? key : "/#{key}" + response = s3_manager.get_object!(params[:bucket_name], key) + send_data response[:body].read, + filename: File.basename(key), + type: response[:content_type] || "application/octet-stream", + disposition: "inline" + rescue Aws::S3::Errors::NoSuchKey + render plain: "File not found", status: 404 + rescue => e + Rails.logger.error "S3 error: #{e.message}" + render plain: "Error retrieving file", status: 500 + end + end + private def site_params diff --git a/app/controllers/swagger_controller.rb b/app/controllers/swagger_controller.rb new file mode 100644 index 00000000..d9f65487 --- /dev/null +++ b/app/controllers/swagger_controller.rb @@ -0,0 +1,5 @@ +class SwaggerController < ApplicationController + layout false + def ui + end +end diff --git a/app/javascript/controllers/audit_export_controller.js b/app/javascript/controllers/audit_export_controller.js new file mode 100644 index 00000000..a6a8719c --- /dev/null +++ b/app/javascript/controllers/audit_export_controller.js @@ -0,0 +1,35 @@ +import {Controller} from "@hotwired/stimulus" + +export default class extends Controller { + static targets = ["button", "preloader", "documentList"] + + static values = { + siteId: Number, + } + + async createReport() { + try { + this.buttonTarget.classList.add("hidden"); + this.preloaderTarget.classList.remove("hidden"); + const response = await fetch(`/sites/${this.siteIdValue}/create_workflow_audit_report`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-CSRF-Token": document.querySelector('meta[name="csrf-token"]').content, + "Accept": "application/json" + }, + }) + if (response.ok) { + const jsonSummary = await response.json() + this.documentListTarget.outerHTML = jsonSummary.html; + this.preloaderTarget.classList.add('hidden') + this.buttonTarget.classList.remove("hidden"); + } else { + throw new Error("Response was not OK") + } + } catch (error) { + console.error("Error summarizing document:", error) + this.preloaderTarget.classList.add('hidden') + } + } +} diff --git a/app/javascript/controllers/index.js b/app/javascript/controllers/index.js index 38841ada..19d6daec 100644 --- a/app/javascript/controllers/index.js +++ b/app/javascript/controllers/index.js @@ -4,6 +4,9 @@ import { application } from "./application" +import AuditExportController from "./audit_export_controller" +application.register("audit-export", AuditExportController) + import BulkEditController from "./bulk_edit_controller" application.register("bulk-edit", BulkEditController) diff --git a/app/javascript/swagger.js b/app/javascript/swagger.js new file mode 100644 index 00000000..d3ce8279 --- /dev/null +++ b/app/javascript/swagger.js @@ -0,0 +1,20 @@ +import SwaggerUI from "swagger-ui-dist/swagger-ui-bundle" +import "swagger-ui-dist/swagger-ui.css" + + +const ui = SwaggerUI({ + url: "/api/swagger_doc", + dom_id: "#swagger-ui", + deepLinking: true, + docExpansion: "list", + defaultModelsExpandDepth: 1, + defaultModelExpandDepth: 1, + displayRequestDuration: true, + showExtensions: true, + showCommonExtensions: true, + tryItOutEnabled: false, + persistAuthorization: true, + supportedSubmitMethods: [] +}) + +window.ui = ui; \ No newline at end of file diff --git a/app/models/site.rb b/app/models/site.rb index f4a36f86..7d68d34c 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -54,6 +54,14 @@ class Site < ApplicationRecord validates :primary_url, presence: true, uniqueness: true validate :ensure_safe_url + after_initialize :after_initialize + + def after_initialize + @s3_manager = AwsS3Manager.new + rescue + @s3_manager = nil + end + def has_departments? documents.where.not(department: [nil, ""]).any? end @@ -231,8 +239,61 @@ def process_archive_or_csv(file_path, is_archive) end end + def export_document_audit!(current_user) + assert_s3_manager + bucket_name = Rails.application.config.default_s3_bucket + machine_site_name = name.downcase.gsub(/\W+/, "_") + report_name = "audit_export_#{machine_site_name}_#{Time.now.strftime("%Y-%m-%dT%H-%M-%S")}" + Tempfile.create([report_name, ".csv"]) do |temp_file| + CSV.open(temp_file.path, "wb") do |csv| + csv << Document.column_names + documents.find_each do |record| + csv << record.attributes.values + end + end + metadata = { + "created" => Time.now.iso8601, + "author" => current_user.email + } + @s3_manager.write_file!(bucket_name, "/reports/#{machine_site_name}/#{report_name}.csv", temp_file.path, "text/csv", metadata) + end + end + + def get_document_audit_exports! + assert_s3_manager + bucket_name = Rails.application.config.default_s3_bucket + machine_site_name = name.downcase.gsub(/\W+/, "_") + { + bucket_name: bucket_name, + files: @s3_manager.get_files!(bucket_name, "/reports/#{machine_site_name}") + } + end + + def get_document_audit_link_hashes! + assert_s3_manager + export_links = [] + report_data = get_document_audit_exports! + if report_data[:files].present? + report_data[:files].each do |s3_object| + metadata = @s3_manager.get_metadata!(report_data[:bucket_name], s3_object.key) + export_links << { + url: Rails.application.routes.url_helpers.workflow_audit_report_site_path(self, report_data[:bucket_name], s3_object.key), + created: metadata.has_key?("created") ? metadata["created"] : "", + author: metadata.has_key?("author") ? metadata["author"] : "" + } + end + end + export_links + end + private + def assert_s3_manager + if @s3_manager.nil? + raise StandardError.new("Failed to connect to AWS environment (AwsS3Manager failed to initialize).") + end + end + def attributes_from(data) { document_category: data[:predicted_category] || data[:document_category], diff --git a/app/services/aws_s3_manager.rb b/app/services/aws_s3_manager.rb new file mode 100644 index 00000000..ef01d830 --- /dev/null +++ b/app/services/aws_s3_manager.rb @@ -0,0 +1,53 @@ +class AwsS3Manager + def initialize + @s3_client = if Rails.env.development? + Aws::S3::Client.new( + endpoint: "http://localhost:4566", + account_id: "none", + access_key_id: "none", + secret_access_key: "none", + region: "us-east-1", + force_path_style: true, + stub_responses: false + ) + else + Aws::S3::Client.new + end + end + + def write_file!(bucket_name, key, file_path, content_type = nil, metadata = {}) + File.open(file_path, "rb") do |file| + @s3_client.put_object( + bucket: bucket_name, + key: key, + body: file, + content_type: content_type || "application/octet-stream", + metadata: metadata + ) + end + end + + def get_files!(bucket_name, key) + prefix = key.end_with?("/") ? key : "#{key}/" + params = { + bucket: bucket_name, + prefix: prefix, + max_keys: 100 + } + response = @s3_client.list_objects_v2(params) + response.contents.sort_by(&:last_modified).reverse + end + + def get_metadata!(bucket_name, key) + head_response = @s3_client.head_object( + bucket: bucket_name, + key: key + ) + return {} unless head_response.metadata.present? + head_response.metadata + end + + def get_object!(bucket_name, key) + @s3_client.get_object(bucket: bucket_name, key: key) + end +end diff --git a/app/views/documents/_audit_export_list.html.erb b/app/views/documents/_audit_export_list.html.erb new file mode 100644 index 00000000..63db1b92 --- /dev/null +++ b/app/views/documents/_audit_export_list.html.erb @@ -0,0 +1,30 @@ +<% if error %> +
+ There was an error retrieving your audit exports. Here are the details:
<%= error %> +
+<% else %> + + + + + + + + + + <% export_links.each do |link| %> + + + + + + + <% end %> + +
Link to ExportDate CreatedAuthor
+ + + Download Report + + <%= link[:created] %><%= link[:author] %>
+<% end %> \ No newline at end of file diff --git a/app/views/documents/audit_exports.html.erb b/app/views/documents/audit_exports.html.erb new file mode 100644 index 00000000..4c9b3c7a --- /dev/null +++ b/app/views/documents/audit_exports.html.erb @@ -0,0 +1,58 @@ +<% content_for :head do %> + <%= tag.meta name: "site-id", content: @site.id %> +<% end %> +
+
+
+ +
+
+
+

<%= @site.location %>: <%= @site.name %>

+
+ <%= link_to site_documents_path(@site), role: "tab", class: "tab" do %> + Documents + <% end %> + <%= link_to insights_site_documents_path(@site), role: "tab", class: "tab" do %> + Insights + <% end %> + <%= link_to audit_exports_site_documents_path(@site), role: "tab", class: "tab tab-active" do %> + Audit Exports + <% end %> +
+
+
+
+
+

Audit Exports

+
+

Create a downloadable CSV export of your audit progress.

+ + + <%= render partial: "documents/audit_export_list", locals: { export_links: @export_links, error: @error_message } %> +
+
+

Use the API

+

Use the following RESTful endpoint to get your audit history.

+
+ 
curl -X 'GET' \
+          '<%= request.base_url %>/api/sites/1/documents?page=0&items_per_page=100' \
+          -H 'accept: application/json' \
+          -H 'authorization: Basic [Your base64 encoded credentials]'
+
+

Read the <%= link_to api_docs_path, class: "text-primary" do %>full API documentation.<% end %>

+
+
+
+
+
+
+
diff --git a/app/views/documents/index.html.erb b/app/views/documents/index.html.erb index 474a4088..58640ccc 100644 --- a/app/views/documents/index.html.erb +++ b/app/views/documents/index.html.erb @@ -16,9 +16,12 @@ <%= link_to site_documents_path(@site), role: "tab", class: "tab tab-active" do %> Documents <% end %> - <%= link_to insights_site_path(@site), role: "tab", class: "tab" do %> + <%= link_to insights_site_documents_path(@site), role: "tab", class: "tab" do %> Insights <% end %> + <%= link_to audit_exports_site_documents_path(@site), role: "tab", class: "tab" do %> + Audit Exports + <% end %> diff --git a/app/views/sites/insights.html.erb b/app/views/documents/insights.html.erb similarity index 93% rename from app/views/sites/insights.html.erb rename to app/views/documents/insights.html.erb index 2aa98476..80b7bf04 100644 --- a/app/views/sites/insights.html.erb +++ b/app/views/documents/insights.html.erb @@ -11,7 +11,7 @@

Filter View

- <%= form_tag insights_site_path(@site), method: :get, class: "px-4 py-4", data: { action: "submit->filter#submitForm" } do %> + <%= form_tag insights_site_documents_path(@site), method: :get, class: "px-4 py-4", data: { action: "submit->filter#submitForm" } do %>
diff --git a/app/views/layouts/swagger.html.erb b/app/views/layouts/swagger.html.erb deleted file mode 100644 index d0653682..00000000 --- a/app/views/layouts/swagger.html.erb +++ /dev/null @@ -1,43 +0,0 @@ - - - - - ASAP PDF API Documentation - - - - -
- - - - - diff --git a/app/views/swagger/ui.html.erb b/app/views/swagger/ui.html.erb new file mode 100644 index 00000000..defcaf07 --- /dev/null +++ b/app/views/swagger/ui.html.erb @@ -0,0 +1,32 @@ + + + + + ASAP PDF API Documentation + <%= javascript_include_tag "swagger", "data-turbo-track": "reload", defer: true %> + <%= stylesheet_link_tag "swagger", "data-turbo-track": "reload" %> + + + +
+ + diff --git a/bin/setup-localstack b/bin/setup-localstack index 45f7d223..1c602a12 100755 --- a/bin/setup-localstack +++ b/bin/setup-localstack @@ -2,24 +2,27 @@ # Wait for LocalStack to be ready echo "Waiting for LocalStack to be ready..." -while ! curl -s "http://localhost:4566/_localstack/health" | grep -q '"s3": "running"'; do - sleep 1 +while ! curl -s "http://localstack:4566/_localstack/health" | grep -q '"s3".*"available"'; do + echo "S3 not ready yet, waiting..." + sleep 2 done +echo "LocalStack S3 is ready!" # Create the bucket if it doesn't exist +# Must match value in config/environments/development.rb, config.default_s3_bucket. echo "Creating S3 bucket..." -aws --endpoint-url=http://localhost:4566 s3 mb s3://cfa-aistudio-asap-pdf +aws --endpoint-url=http://localstack:4566 s3 mb s3://asap-pdf-staging-documents # Enable versioning on the bucket echo "Enabling bucket versioning..." -aws --endpoint-url=http://localhost:4566 s3api put-bucket-versioning \ - --bucket cfa-aistudio-asap-pdf \ +aws --endpoint-url=http://localstack:4566 s3api put-bucket-versioning \ + --bucket asap-pdf-staging-documents \ --versioning-configuration Status=Enabled # Set bucket policy to allow public access (for development only) echo "Setting bucket policy..." -aws --endpoint-url=http://localhost:4566 s3api put-bucket-policy \ - --bucket cfa-aistudio-asap-pdf \ +aws --endpoint-url=http://localstack:4566 s3api put-bucket-policy \ + --bucket asap-pdf-staging-documents \ --policy '{ "Version": "2012-10-17", "Statement": [ @@ -28,7 +31,7 @@ aws --endpoint-url=http://localhost:4566 s3api put-bucket-policy \ "Effect": "Allow", "Principal": "*", "Action": "s3:*", - "Resource": "arn:aws:s3:::cfa-aistudio-asap-pdf/*" + "Resource": "arn:aws:s3:::asap-pdf-staging-documents/*" } ] }' diff --git a/bin/setup_python_components b/bin/setup_python_components deleted file mode 100755 index 926e48e3..00000000 --- a/bin/setup_python_components +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -declare -a arr=("GEMINI_KEY" "ANTHROPIC_KEY") - -for item in "${arr[@]}"; do - read -p "Enter value for $item: " key - docker exec asap_setup aws secretsmanager create-secret \ - --endpoint-url=http://localstack:4566 \ - --name $item \ - --description "LocalStack secret $key" \ - --secret-string $key \ - --region 'us-east-1' - echo "Set secret value for $item" -done - -echo "Python components should be ready to run!" \ No newline at end of file diff --git a/config/credentials/production.yml.enc b/config/credentials/production.yml.enc new file mode 100644 index 00000000..2450c765 --- /dev/null +++ b/config/credentials/production.yml.enc @@ -0,0 +1 @@ +t+sp+zLTSKdE0nQaoq74+EZ2GZghyXnVDC9iZKXFjJzdCpzRBouHaHYe2hcTvK88xhrE2mMEvRC72q3fB9ON6Gu+jwPAaR/kyKZLJuMvjaer1Kr57GAEyHmALOqtCHKTRVt9i1SU+YeiXrZ5vrUF6ZXqfISsP5W8OeILAXUv0h3KGq7jenDQAclQTNIrhHG6WgxYdnyrzuFgBWZtYkZUVaBMgX8BAtwN8QX4qXDG31zk3TOt893+3Da/IwiTqMMVphvl1i9qAtQrdwS03sibXTfUVTluv75cU2XDxMvPro7Ws4vgKmwFjiUJKx7IXaZQxBARJD0EOzkhFjiQOfdW2s694Or/uJGtO/nnWjOUrP+8+NWigzOxatjueikuNEAJ//lkTwMX4ni3AZrbC5KrShMWWNrfLAkYpjCthOc9tuPAgRJlrhW/j7x1cGNE3+YOSz4ze/WcpR9SwZfu9SosBJOp1MhJFGww5wXCc66bYGifamnHD+Qr7bx8--4EiPuW5dKUPW0HGF--wVQ1fNnNfCAxI6nvxUne5w== \ No newline at end of file diff --git a/config/database.yml b/config/database.yml index a064c8dc..5584a509 100644 --- a/config/database.yml +++ b/config/database.yml @@ -4,6 +4,7 @@ default: &default prepared_statements: false pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + development: <<: *default host: localhost diff --git a/config/environments/development.rb b/config/environments/development.rb index 0cb5b88b..66b897f8 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -38,4 +38,15 @@ config.action_controller.raise_on_missing_callback_actions = true config.hosts = nil + + config.default_s3_bucket = "asap-pdf-staging-documents" + + config.local_secret_names = { + asap_api_user: "asap-pdf/staging/RAILS_API_USER", + asap_api_password: "asap-pdf/staging/RAILS_API_PASSWORD", + google_api: "asap-pdf/staging/GOOGLE_AI_KEY", + anthropic_api: "asap-pdf/staging/ANTHROPIC_KEY", + google_eval_service_account: "asap-pdf/staging/GOOGLE_SERVICE_ACCOUNT", + google_eval_sheet_id: "asap-pdf/staging/GOOGLE_SHEET_ID_EVALUATION" + } end diff --git a/config/environments/production.rb b/config/environments/production.rb index 1d825d3e..b559ef3d 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -43,4 +43,6 @@ config.i18n.fallbacks = true config.active_record.dump_schema_after_migration = false config.active_record.attributes_for_inspect = [:id] + + config.default_s3_bucket = "asap-pdf-prod-documents" end diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 83af0209..f7afdaa1 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -43,4 +43,6 @@ config.i18n.fallbacks = true config.active_record.dump_schema_after_migration = false config.active_record.attributes_for_inspect = [:id] + + config.default_s3_bucket = "asap-pdf-staging-documents" end diff --git a/config/environments/test.rb b/config/environments/test.rb index 1446aa10..615b4cd3 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -17,4 +17,15 @@ config.active_support.deprecation = :stderr config.action_controller.raise_on_missing_callback_actions = true + + config.default_s3_bucket = "asap-pdf-staging-documents" + + config.local_secret_names = { + asap_api_user: "asap-pdf/staging/RAILS_API_USER", + asap_api_password: "asap-pdf/staging/RAILS_API_PASSWORD", + google_api: "asap-pdf/staging/GOOGLE_AI_KEY", + anthropic_api: "asap-pdf/staging/ANTHROPIC_KEY", + google_eval_service_account: "asap-pdf/staging/GOOGLE_SERVICE_ACCOUNT", + google_eval_sheet_id: "asap-pdf/staging/GOOGLE_SHEET_ID_EVALUATION" + } end diff --git a/config/routes.rb b/config/routes.rb index 05218724..19c45da8 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,3 +1,5 @@ +require "sidekiq/web" + Rails.application.routes.draw do devise_for :users, controllers: { sessions: "users/sessions", @@ -13,7 +15,8 @@ resources :sites do member do - get :insights + get "workflow_audit_report/:bucket_name/*key", to: "sites#workflow_audit_report", format: false, as: :workflow_audit_report + post :create_workflow_audit_report end resources :documents do member do @@ -22,6 +25,8 @@ end collection do patch :batch_update + get :insights + get :audit_exports end end end @@ -45,7 +50,8 @@ end mount AsapPdf::API => "/api" - get "api-docs", to: "api_docs#index" + + get "/api-docs", to: "swagger#ui" resource :configuration, only: [:edit, :update] diff --git a/package.json b/package.json index 643db994..5e578adb 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "chartkick": "^5.0.1", "daisyui": "^4.12.23", "postcss": "^8.5.1", + "swagger-ui-dist": "^5.27.1", "tailwindcss": "^3.4.1" } } diff --git a/spec/factories/document_inferences.rb b/spec/factories/document_inferences.rb new file mode 100644 index 00000000..d652d90e --- /dev/null +++ b/spec/factories/document_inferences.rb @@ -0,0 +1,11 @@ +FactoryBot.define do + factory :document_inference do + creation_date { Time.current } + inference_type { "exception:is_application" } + inference_value { "True" } + inference_confidence { 0.85 } + inference_reason { "This is an event flyer for a for a croquet party." } + is_active { true } + document + end +end diff --git a/spec/features/document_spec.rb b/spec/features/document_spec.rb index cf5e629b..44de145e 100644 --- a/spec/features/document_spec.rb +++ b/spec/features/document_spec.rb @@ -507,7 +507,7 @@ click_button "Apply Filters" end sleep(1) - assert_match "sites/#{site.id}/insights?category=Agreement&department=Public+Transportation", current_url + assert_match "sites/#{site.id}/documents/insights?category=Agreement&department=Public+Transportation", current_url # Look for general content. within("#insights") do expect(page).to have_content "\nTYPE NEEDS DECISION IN REVIEW DONE TOTAL\nAgreement 1 0 0 1" @@ -527,7 +527,7 @@ teahouse_doc.save market_doc.accessibility_recommendation = Document::ARCHIVE_DECISION market_doc.save - visit "/sites/#{site.id}/insights?category=Agreement&department=Public+Transportation&accessibility_decision=Remediate" + visit "/sites/#{site.id}/documents/insights?category=Agreement&department=Public+Transportation&accessibility_decision=Remediate" within("#insights #chart-modification-year") do find(".dropdown .btn").click click_link "2018-2023" @@ -535,7 +535,7 @@ sleep(1) assert_match "sites/#{site.id}/documents?accessibility_decision=Remediate&category=Agreement&department=Public+Transportation&end_date=2023-12-31&start_date=2018-01-01", current_url - visit "/sites/#{site.id}/insights?category=Notice&department=Parks+and+Recreation&status=Archive" + visit "/sites/#{site.id}/documents/insights?category=Notice&department=Parks+and+Recreation&status=Archive" within("#insights #chart-decision") do find(".dropdown .btn").click click_link "Archive" @@ -609,9 +609,21 @@ click_button "AI Exception Check" expect(page).to have_no_content "Feedback on AI Response:" end - DocumentInference.create(document_id: doc.id, inference_type: "summary", inference_value: "A lovely example of accessible PDF practices.", is_active: true) - DocumentInference.create(inference_type: "exception:is_application", inference_value: "True", inference_reason: "This is not used as an application or means of participation in government services.", document_id: doc.id, is_active: true) - DocumentInference.create(inference_type: "exception:is_archival", inference_value: "True", inference_reason: "This thing was made in 1988 and hasn't been opened since then.", document_id: doc.id, is_active: true) + DocumentInference.new(document_id: doc.id, inference_type: "summary", inference_value: "A lovely example of accessible PDF practices.").save + DocumentInference.create( + inference_type: "exception:is_application", + inference_value: "True", + inference_reason: "This is not used as an application or means of participation in government services.", + document_id: doc.id, + is_active: true + ) + DocumentInference.create( + inference_type: "exception:is_archival", + inference_value: "True", + inference_reason: "This thing was made in 1988 and hasn't been opened since then.", + document_id: doc.id, + is_active: true + ) visit "/" click_link "City of Denver" within("#document-list") do @@ -676,4 +688,21 @@ expect(FeedbackItem.count).to eq 5 end end + + it "exports documents" do + site = Site.create(name: "City of Denver", location: "Colorado", primary_url: "https://denvergov.org") + @current_user.site = site + @current_user.save! + Document.create(url: "http://denvergov.org/docs/ex.ample.pdf", file_name: "ex.ample.pdf", document_category: "Agenda", site_id: site.id) + visit "/" + click_link "City of Denver" + within("#document-list #document-tabs") do + click_link "Audit Exports" + sleep(1) + end + assert_match "sites/#{site.id}/documents/audit_exports", current_url + expect(page).to have_content "Create Audit Export" + expect(page).to have_content "Use the following RESTful endpoint to get your audit history." + expect(page).to have_content "authorization: Basic [Your base64 encoded credentials]" + end end diff --git a/spec/requests/api/sites_spec.rb b/spec/requests/api/sites_spec.rb index de9737c0..96e14bd8 100644 --- a/spec/requests/api/sites_spec.rb +++ b/spec/requests/api/sites_spec.rb @@ -7,13 +7,13 @@ def app AsapPdf::API end - def auth_headers - user = User.last + def auth_headers user encoded_credentials = ActionController::HttpAuthentication::Basic.encode_credentials(user.email, "password") {"HTTP_AUTHORIZATION" => encoded_credentials} end - let!(:user) { create(:user, :site_admin) } + let!(:admin_user) { create(:user, :site_admin) } + let!(:user) { create(:user) } describe "GET /sites" do let!(:sites) { create_list(:site, 3) } @@ -22,14 +22,25 @@ def auth_headers expect(last_response.status).to eq(401) end - it "returns all sites" do - get "/sites", {}, auth_headers + it "returns all accessible sites" do + get "/sites", {}, auth_headers(admin_user) expect(last_response.status).to eq(200) expect(JSON.parse(last_response.body).length).to eq(3) + + get "/sites", {}, auth_headers(user) + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body).length).to eq(0) + + user.site = sites[0] + user.save! + + get "/sites", {}, auth_headers(user) + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body).length).to eq(1) end it "returns sites with correct structure" do - get "/sites", {}, auth_headers + get "/sites", {}, auth_headers(admin_user) json_response = JSON.parse(last_response.body) first_site = json_response.first @@ -42,135 +53,75 @@ def auth_headers end end - describe "GET /sites/:id" do + describe "GET /sites/:id/documents" do let!(:site) { create(:site) } - context "when the site exists" do - it "returns the requested site" do - get "/sites/#{site.id}", {}, auth_headers - expect(last_response.status).to eq(200) - - json_response = JSON.parse(last_response.body) - expect(json_response["id"]).to eq(site.id) - expect(json_response["name"]).to eq(site.name) - expect(json_response["location"]).to eq(site.location) - expect(json_response["primary_url"]).to eq(site.primary_url) - end - end + let!(:document) { create_list(:document, 10, site: site) } - context "when the site does not exist" do - it "returns 404 not found" do - get "/sites/0", {}, auth_headers - expect(last_response.status).to eq(404) - end - end - end - - describe "POST /sites/:id/documents" do - let!(:site) { create(:site) } - let(:timestamp) { Time.current } - let(:valid_documents) do - [ - {url: "https://example.com/doc1.pdf", modification_date: timestamp, document_category: "Brochure"}, - {url: "https://example.com/doc2.pdf", modification_date: timestamp, document_category: "Brochure"} - ] + it "blocks access to anonymous users" do + get "/sites/#{site.id}/documents" + expect(last_response.status).to eq(401) end - context "when the site exists" do - it "blocks access to anonymous users" do - post "/sites/#{site.id}/documents", {documents: valid_documents} - expect(last_response.status).to eq(401) - end - - it "creates new documents for new URLs" do - expect { - post "/sites/#{site.id}/documents", {documents: valid_documents}, auth_headers - }.to change(Document, :count).by(2) - - expect(last_response.status).to eq(201) - - json_response = JSON.parse(last_response.body) - expect(json_response["documents"].length).to eq(2) - - first_doc = json_response["documents"].first - expect(first_doc).to include( - "id", - "url", - "document_status", - "s3_path" - ) - expect(first_doc["url"]).to eq(valid_documents.first[:url]) - expect(first_doc["document_status"]).to eq("discovered") - expect(first_doc["s3_path"]).to include(site.s3_endpoint_prefix) - end - - it "updates existing documents when modification_date changes" do - existing_doc = site.documents.create!( - url: valid_documents.first[:url], - modification_date: 1.day.ago, - file_name: "doc1.pdf", - document_status: "discovered", - document_category: "Brochure" - ) + it "returns all accessible documents" do + get "/sites/#{site.id}/documents", {}, auth_headers(admin_user) + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body)["documents"].length).to eq(10) - expect { - post "/sites/#{site.id}/documents", {documents: valid_documents}, auth_headers - }.to change(Document, :count).by(1) # Only creates one new document + get "/sites/#{site.id}/documents", {}, auth_headers(user) + expect(last_response.status).to eq(401) - expect(last_response.status).to eq(201) + user.site = site + user.save! - existing_doc.reload - expect(existing_doc.document_status).to eq("discovered") - expect(existing_doc.modification_date).to be_within(1.second).of(timestamp) - end - - it "doesn't modify existing documents when modification_date hasn't changed" do - existing_doc = site.documents.create!( - url: valid_documents.first[:url], - modification_date: timestamp, - file_name: "doc1.pdf", - document_status: "discovered", - document_category: "Brochure" - ) + get "/sites/#{site.id}/documents", {}, auth_headers(user) + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body)["documents"].length).to eq(10) + end - expect { - post "/sites/#{site.id}/documents", {documents: valid_documents}, auth_headers - }.to change(Document, :count).by(1) # Only creates one new document + it "paginates" do + get "/sites/#{site.id}/documents", {page: 0, items_per_page: 2}, auth_headers(admin_user) + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body)["documents"].length).to eq(2) + get "/sites/#{site.id}/documents", {page: 4, items_per_page: 2}, auth_headers(admin_user) + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body)["documents"].length).to eq(2) + get "/sites/#{site.id}/documents", {page: 5, items_per_page: 2}, auth_headers(admin_user) + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body)["documents"].length).to eq(0) + end + end - expect(last_response.status).to eq(201) + describe "GET /documents/:id/document_inference" do + let!(:site) { create(:site) } + let!(:document) { create(:document, site: site) } + let!(:document_inference) { create_list(:document_inference, 4, document: document) } - existing_doc.reload - expect(existing_doc.document_status).to eq("discovered") - end + it "blocks access to anonymous users" do + get "/documents/#{document.id}/document_inference" + expect(last_response.status).to eq(401) end - context "when the site does not exist" do - it "returns 404 not found" do - post "/sites/0/documents", {documents: valid_documents}, auth_headers - expect(last_response.status).to eq(404) - end - end + it "returns all accessible document inferences" do + get "/documents/#{document.id}/document_inference", {}, auth_headers(admin_user) + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body)["document_inferences"].length).to eq(4) - context "with invalid parameters" do - it "returns 400 bad request when documents is missing" do - post "/sites/#{site.id}/documents", {}, auth_headers - expect(last_response.status).to eq(400) - end + get "/documents/#{document.id}/document_inference", {}, auth_headers(user) + expect(last_response.status).to eq(401) - it "returns 400 bad request when documents is not an array" do - post "/sites/#{site.id}/documents", {documents: "not_an_array"}, auth_headers - expect(last_response.status).to eq(400) - end + user.site = site + user.save! - it "returns 400 bad request when document is missing required fields" do - post "/sites/#{site.id}/documents", {documents: [{url: "https://example.com/doc.pdf"}]}, auth_headers - expect(last_response.status).to eq(400) - end + get "/documents/#{document.id}/document_inference", {}, auth_headers(user) + expect(last_response.status).to eq(200) + expect(JSON.parse(last_response.body)["document_inferences"].length).to eq(4) end end describe "POST /documents/:id/inference" do let(:timestamp) { Time.current } - let!(:document) { create(:document) } + let!(:site) { create(:site) } + let!(:document) { create(:document, site: site) } let(:inference) { {inference_type: "exception", result: {is_archival: "True", why_archival: "This document is in a special archival section."}} } let(:inference_update) { {inference_type: "exception", result: {is_archival: "True", why_archival: "This document is in a special archival section.", is_application: "True", why_application: "Test 123"}} } @@ -181,13 +132,24 @@ def auth_headers end it "creates new inferences" do expect { - post "/documents/#{document.id}/inference", inference, auth_headers + post "/documents/#{document.id}/inference", inference, auth_headers(admin_user) }.to change(DocumentInference, :count).by(1) expect(document.document_inferences.count).to eq(1) expect { - post "/documents/#{document.id}/inference", inference_update, auth_headers + post "/documents/#{document.id}/inference", inference_update, auth_headers(admin_user) }.to change(DocumentInference, :count).by(2) expect(document.document_inferences.count).to eq(3) + + post "/documents/#{document.id}/inference", inference_update, auth_headers(user) + expect(last_response.status).to eq(401) + expect(document.document_inferences.count).to eq(3) + + user.site = site + user.save! + + post "/documents/#{document.id}/inference", inference_update, auth_headers(user) + expect(last_response.status).to eq(201) + expect(document.document_inferences.count).to eq(5) end end end diff --git a/terraform/modules/ecs/main.tf b/terraform/modules/ecs/main.tf index 9fb36c67..df5f9833 100644 --- a/terraform/modules/ecs/main.tf +++ b/terraform/modules/ecs/main.tf @@ -16,8 +16,8 @@ module "fargate_service" { memory = 2048 container_port = 3000 - execution_policies = [aws_iam_policy.ecs_task_secrets_policy.arn, aws_iam_policy.ecs_s3_access.arn] - task_policies = [aws_iam_policy.ecs_task_lambda_invoke_policy.arn] + execution_policies = [aws_iam_policy.ecs_task_secrets_policy.arn] + task_policies = [aws_iam_policy.ecs_task_lambda_invoke_policy.arn, aws_iam_policy.ecs_s3_access.arn] enable_execute_command = true create_version_parameter = true diff --git a/yarn.lock b/yarn.lock index bc21c61b..3a2266a4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -235,6 +235,11 @@ resolved "https://registry.yarnpkg.com/@rails/actioncable/-/actioncable-7.2.201.tgz#bfb3da01b3e2462f5a18f372c52dedd7de76037f" integrity sha512-wsTdWoZ5EfG5k3t7ORdyQF0ZmDEgN4aVPCanHAiNEwCROqibSZMXXmCbH7IDJUVri4FOeAVwwbPINI7HVHPKBw== +"@scarf/scarf@=1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@scarf/scarf/-/scarf-1.4.0.tgz#3bbb984085dbd6d982494538b523be1ce6562972" + integrity sha512-xxeapPiUXdZAE3che6f3xogoJPeZgig6omHEy1rIY5WVsB3H2BHNnZH+gHG6x91SCWyQCzWGsuL2Hh3ClO5/qQ== + ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" @@ -945,6 +950,13 @@ supports-preserve-symlinks-flag@^1.0.0: resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== +swagger-ui-dist@^5.27.1: + version "5.27.1" + resolved "https://registry.yarnpkg.com/swagger-ui-dist/-/swagger-ui-dist-5.27.1.tgz#556e77c659752e99621ac61ad5ef6cb0832279e7" + integrity sha512-oGtpYO3lnoaqyGtlJalvryl7TwzgRuxpOVWqEHx8af0YXI+Kt+4jMpLdgMtMcmWmuQ0QTCHLKExwrBFMSxvAUA== + dependencies: + "@scarf/scarf" "=1.4.0" + tailwindcss@^3.4.1: version "3.4.17" resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.4.17.tgz#ae8406c0f96696a631c790768ff319d46d5e5a63"