From 57b212d14942e2ff9e5a0928b6032523616cf067 Mon Sep 17 00:00:00 2001 From: Denis Tataurov Date: Tue, 18 Jul 2017 19:40:50 +0300 Subject: [PATCH] Add tests; Update ex_aws --- README.md | 40 +++++++++++++++++++++++---------- config/config.exs | 3 ++- config/test.exs | 7 ++++++ lib/exfile_s3/backend.ex | 40 ++++++++++++++++++--------------- lib/exfile_s3/client.ex | 32 ++++++++++++++++++++++++++ lib/exfile_s3/config.ex | 8 +++++++ lib/exfile_s3/s3.ex | 15 ------------- mix.exs | 19 ++++++++-------- mix.lock | 26 +++++++++++---------- test/exfile_s3/backend_test.exs | 16 +++++++++++++ 10 files changed, 139 insertions(+), 67 deletions(-) create mode 100644 config/test.exs create mode 100644 lib/exfile_s3/client.ex create mode 100644 lib/exfile_s3/config.ex delete mode 100644 lib/exfile_s3/s3.ex create mode 100644 test/exfile_s3/backend_test.exs diff --git a/README.md b/README.md index 8707ac0..402a37a 100644 --- a/README.md +++ b/README.md @@ -6,29 +6,45 @@ An AWS S3 adapter for [exfile](https://github.com/keichan34/exfile), using the [ If [available in Hex](https://hex.pm/docs/publish), the package can be installed as: - 1. Add exfile_s3 to your list of dependencies in `mix.exs`: + 1. Add `exfile_s3` to your list of dependencies in `mix.exs`: + ```elixir def deps do - [{:exfile_s3, "~> 0.0.1"}] + [{:exfile_s3, "~> 0.0.3"}] end + ``` + 2. Ensure `exfile_s3` is started before your application: - 2. Ensure exfile_s3 is started before your application: - + ```elixir def application do [applications: [:exfile_s3]] end + ``` + + 3. Configure `ex_aws` in config.exs: - 3. Configure the backend in config.exs (or environment equivalent) + ```elixir + config :ex_aws, :s3, + access_key_id: System.get_env("AWS_ACCESS_KEY_ID"), + secret_access_key: System.get_env("AWS_SECRET_ACCESS_KEY"), + region: System.get_env("AWS_REGION"), + bucket: System.get_env("AWS_S3_BUCKET") + ``` + 4. Add the `exfile` backend in config.exs (or environment equivalent) + + ```elixir config :exfile, Exfile, backends: %{ - "store": {ExfileS3.Backend, + "store" => { + ExfileS3.Backend, + hasher: Exfile.Hasher.Random, + cdn_host: "CDN Host" + }, + "cache" => { + ExfileS3.Backend, hasher: Exfile.Hasher.Random, - access_key_id: "AWS Access Key Id", - secret_access_key: "AWS Secret Access Key", - s3_prefix: "bucket prefix", - bucket_region: "Bucket Region", - cdn_host: "CDN Host", - bucket: "Name of the bucket to store files" + cdn_host: "CDN Host" } } + ``` diff --git a/config/config.exs b/config/config.exs index 6d949a9..bfa468a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -27,4 +27,5 @@ use Mix.Config # Configuration from the imported file will override the ones defined # here (which is why it is important to import them last). # -# import_config "#{Mix.env}.exs" + +import_config "#{Mix.env}.exs" diff --git a/config/test.exs b/config/test.exs new file mode 100644 index 0000000..c48c0fd --- /dev/null +++ b/config/test.exs @@ -0,0 +1,7 @@ +use Mix.Config + +config :ex_aws, :s3, + access_key_id: System.get_env("AWS_ACCESS_KEY_ID"), + secret_access_key: System.get_env("AWS_SECRET_ACCESS_KEY"), + region: System.get_env("AWS_REGION"), + bucket: System.get_env("AWS_S3_BUCKET") diff --git a/lib/exfile_s3/backend.ex b/lib/exfile_s3/backend.ex index 3c19d51..1cf5abc 100644 --- a/lib/exfile_s3/backend.ex +++ b/lib/exfile_s3/backend.ex @@ -1,14 +1,13 @@ defmodule ExfileS3.Backend do use Exfile.Backend - alias Exfile.LocalFile - alias ExfileS3.S3 - def init(opts) do - _access_key_id = Keyword.get(opts, :access_key_id) || raise(ArgumentError, message: "access_key_id is required.") - _secret_access_key = Keyword.get(opts, :secret_access_key) || raise(ArgumentError, message: "secret_access_key is required.") - _bucket = Keyword.get(opts, :bucket) || raise(ArgumentError, message: "bucket is required.") {:ok, backend} = super(opts) + ExfileS3.Config.get(backend, :access_key_id) || raise(ArgumentError, message: "access_key_id is required.") + ExfileS3.Config.get(backend, :secret_access_key) || raise(ArgumentError, message: "secret_access_key is required.") + ExfileS3.Config.get(backend, :region) || raise(ArgumentError, message: "region is required.") + ExfileS3.Config.get(backend, :bucket) || raise(ArgumentError, message: "bucket is required.") + backend end @@ -21,11 +20,11 @@ defmodule ExfileS3.Backend do end end - def upload(backend, %LocalFile{} = uploadable) do - id = backend.hasher.hash(uploadable) - case LocalFile.open(uploadable) do + def upload(backend, %Exfile.LocalFile{} = uploadable) do + file_id = backend.hasher.hash(uploadable) + case Exfile.LocalFile.open(uploadable) do {:ok, io} -> - perform_upload(backend, id, io) + perform_upload(backend, file_id, io) {:error, reason} -> {:error, reason} end @@ -36,27 +35,32 @@ defmodule ExfileS3.Backend do {:error, reason} -> {:error, reason} iodata -> - S3.put_object(S3.find_config_value(:bucket), path(backend, file_id), iodata) + ExfileS3.Client.put_object(backend, path(backend, file_id), iodata) {:ok, get(backend, file_id)} end end def open(backend, file_id) do - case S3.get_object(S3.find_config_value(:bucket), path(backend, file_id)) do + case ExfileS3.Client.get_object(backend, path(backend, file_id)) do {:ok, %{body: body}} -> {:ok, io} = File.open(body, [:ram, :binary, :read]) - {:ok, %LocalFile{io: io}} + {:ok, %Exfile.LocalFile{io: io}} {:error, reason} -> {:error, reason} end end def delete(backend, file_id) do - S3.delete_object(S3.find_config_value(:bucket), path(backend, file_id)) + case exists?(backend, file_id) do + true -> + ExfileS3.Client.delete_object(backend, path(backend, file_id)) |> elem(0) + false -> + :ok + end end def size(backend, file_id) do - case S3.head_object(S3.find_config_value(:bucket), path(backend, file_id)) do + case ExfileS3.Client.head_object(backend, path(backend, file_id)) do {:ok, %{headers: headers}} -> {"Content-Length", size} = Enum.find(headers, fn({header_name, _}) -> header_name == "Content-Length" end) {size, _} = Integer.parse(size) @@ -67,14 +71,14 @@ defmodule ExfileS3.Backend do end def exists?(backend, file_id) do - case S3.head_object(S3.find_config_value(:bucket), path(backend, file_id)) do + case ExfileS3.Client.head_object(backend, path(backend, file_id)) do {:ok, _} -> true _ -> false end end - def path(_backend, id) do - [S3.find_config_value(:prefix), id] + def path(backend, id) do + [backend.backend_name, id] |> Enum.reject(&is_empty?/1) |> Enum.join("/") end diff --git a/lib/exfile_s3/client.ex b/lib/exfile_s3/client.ex new file mode 100644 index 0000000..75e642c --- /dev/null +++ b/lib/exfile_s3/client.ex @@ -0,0 +1,32 @@ +defmodule ExfileS3.Client do + + def get_object(backend, path) do + backend + |> get_bucket() + |> ExAws.S3.get_object(path) + |> ExAws.request + end + + def put_object(backend, path, iodata) do + backend + |> get_bucket() + |> ExAws.S3.put_object(path, iodata) + |> ExAws.request + end + + def delete_object(backend, path) do + backend + |> get_bucket() + |> ExAws.S3.delete_object(path) + |> ExAws.request + end + + def head_object(backend, path) do + backend + |> get_bucket() + |> ExAws.S3.head_object(path) + |> ExAws.request + end + + defp get_bucket(backend), do: ExfileS3.Config.get(backend, :bucket) +end diff --git a/lib/exfile_s3/config.ex b/lib/exfile_s3/config.ex new file mode 100644 index 0000000..492f8cd --- /dev/null +++ b/lib/exfile_s3/config.ex @@ -0,0 +1,8 @@ +defmodule ExfileS3.Config do + def get(_, :access_key_id), do: s3_config() |> Keyword.get(:access_key_id) + def get(_, :secret_access_key), do: s3_config() |> Keyword.get(:secret_access_key) + def get(_, :bucket), do: s3_config() |> Keyword.get(:bucket) + def get(_, :region), do: s3_config() |> Keyword.get(:region) + + defp s3_config, do: Application.get_env(:ex_aws, :s3) +end diff --git a/lib/exfile_s3/s3.ex b/lib/exfile_s3/s3.ex deleted file mode 100644 index 8f1ae61..0000000 --- a/lib/exfile_s3/s3.ex +++ /dev/null @@ -1,15 +0,0 @@ -defmodule ExfileS3.S3 do - use ExAws.S3.Client - - @valid_options [:access_key_id, :secret_access_key, :prefix, :region, :cdn_host, :bucket] - - def config_root do - %{"store" => {ExfileS3.Backend, s3_config}} = Application.get_env(:exfile, Exfile, []) |> Keyword.get(:backends, %{}) - s3_config = s3_config |> Keyword.take(@valid_options) - [s3: s3_config] - end - - def find_config_value(key) do - config_root |> Keyword.get(:s3) |> Keyword.get(key) - end -end diff --git a/mix.exs b/mix.exs index cabf326..4a55ee9 100644 --- a/mix.exs +++ b/mix.exs @@ -4,13 +4,13 @@ defmodule ExfileS3.Mixfile do def project do [ app: :exfile_s3, - version: "0.0.2", + version: "0.0.3", elixir: "~> 1.2", build_embedded: Mix.env == :prod, start_permanent: Mix.env == :prod, - deps: deps, - description: description, - package: package + deps: deps(), + description: description(), + package: package() ] end @@ -48,11 +48,12 @@ defmodule ExfileS3.Mixfile do # Type "mix help deps" for more examples and options defp deps do [ - {:exfile, "~> 0.3"}, - {:ex_aws, "~> 0.5"}, - {:sweet_xml, "~> 0.6"}, - {:httpoison, "~> 0.8"}, - {:poison, "~> 1.5 or ~> 2.0"} + # {:exfile, "~> 0.3"}, + {:exfile, git: "git@github.com:grabr/exfile.git", branch: "phx-1.3"}, + {:ex_aws, "~> 0.5 or ~> 1.1", optional: true}, + {:sweet_xml, "~> 0.6", optional: true}, + {:httpoison, "~> 0.8", optional: true}, + {:poison, "~> 1.5 or ~> 2.0 or ~> 3.1", optional: true} ] end end diff --git a/mix.lock b/mix.lock index 47956a0..f0ac78b 100644 --- a/mix.lock +++ b/mix.lock @@ -1,12 +1,14 @@ -%{"certifi": {:hex, :certifi, "0.4.0", "a7966efb868b179023618d29a407548f70c52466bf1849b9e8ebd0e34b7ea11f", [:rebar3], []}, - "ex_aws": {:hex, :ex_aws, "0.5.0", "6ca02f1e8fe8340aa2eee66d9f08efcd6ff1f9f4ef7264669d0756e0b3917218", [:mix], [{:jsx, "~> 2.5", [hex: :jsx, optional: true]}, {:poison, "~> 1.2 or ~> 2.0", [hex: :poison, optional: true]}, {:httpoison, "~> 0.8", [hex: :httpoison, optional: true]}, {:sweet_xml, "~> 0.5", [hex: :sweet_xml, optional: true]}]}, - "exfile": {:hex, :exfile, "0.3.3", "0c092011e42d47be4bde65ba6edfc76c13ef54fa221a50a1a311e2301b3335d3", [:mix], [{:poison, "~> 1.5 or ~> 2.0", [hex: :poison, optional: true]}, {:plug, "~> 1.0", [hex: :plug, optional: false]}, {:phoenix_html, "~> 2.3", [hex: :phoenix_html, optional: true]}, {:phoenix, "~> 1.1.4", [hex: :phoenix, optional: true]}, {:ecto, "~> 1.0", [hex: :ecto, optional: true]}]}, - "hackney": {:hex, :hackney, "1.6.1", "ddd22d42db2b50e6a155439c8811b8f6df61a4395de10509714ad2751c6da817", [:rebar3], [{:ssl_verify_fun, "1.1.0", [hex: :ssl_verify_fun, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:metrics, "1.0.1", [hex: :metrics, optional: false]}, {:idna, "1.2.0", [hex: :idna, optional: false]}, {:certifi, "0.4.0", [hex: :certifi, optional: false]}]}, - "httpoison": {:hex, :httpoison, "0.9.0", "68187a2daddfabbe7ca8f7d75ef227f89f0e1507f7eecb67e4536b3c516faddb", [:mix], [{:hackney, "~> 1.6.0", [hex: :hackney, optional: false]}]}, - "idna": {:hex, :idna, "1.2.0", "ac62ee99da068f43c50dc69acf700e03a62a348360126260e87f2b54eced86b2", [:rebar3], []}, - "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []}, - "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []}, - "plug": {:hex, :plug, "1.1.5", "de5645c18170415a72b18cc3d215c05321ddecac27a15acb923742156e98278b", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, optional: true]}]}, - "poison": {:hex, :poison, "2.2.0", "4763b69a8a77bd77d26f477d196428b741261a761257ff1cf92753a0d4d24a63", [:mix], []}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.0", "edee20847c42e379bf91261db474ffbe373f8acb56e9079acb6038d4e0bf414f", [:rebar, :make], []}, - "sweet_xml": {:hex, :sweet_xml, "0.6.1", "a56f235171f35a32807ce44798dedc748ce249fca574674fecd29c1321cab0de", [:mix], []}} +%{"certifi": {:hex, :certifi, "1.2.1", "c3904f192bd5284e5b13f20db3ceac9626e14eeacfbb492e19583cf0e37b22be", [:rebar3], [], "hexpm"}, + "ex_aws": {:hex, :ex_aws, "1.1.3", "08f3905e75ff0bb84f3e99156662e1620ba63e55a337eed6ab91d38d4f8f5a10", [:mix], [{:configparser_ex, "~> 0.2.1", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.0.6", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"}, + "exfile": {:git, "git@github.com:grabr/exfile.git", "c10ee279aa54df5c8251c2b40ff2cc22e7b45406", [branch: "phx-1.3"]}, + "hackney": {:hex, :hackney, "1.8.6", "21a725db3569b3fb11a6af17d5c5f654052ce9624219f1317e8639183de4a423", [:rebar3], [{:certifi, "1.2.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.0.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, + "httpoison": {:hex, :httpoison, "0.12.0", "8fc3d791c5afe6beb0093680c667dd4ce712a49d89c38c3fe1a43100dd76cf90", [:mix], [{:hackney, "~> 1.8.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, + "idna": {:hex, :idna, "5.0.2", "ac203208ada855d95dc591a764b6e87259cb0e2a364218f215ad662daa8cd6b4", [:rebar3], [{:unicode_util_compat, "0.2.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, + "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, + "mime": {:hex, :mime, "1.1.0", "01c1d6f4083d8aa5c7b8c246ade95139620ef8effb009edde934e0ec3b28090a", [:mix], [], "hexpm"}, + "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"}, + "plug": {:hex, :plug, "1.3.5", "7503bfcd7091df2a9761ef8cecea666d1f2cc454cbbaf0afa0b6e259203b7031", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1", [hex: :cowboy, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm"}, + "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], [], "hexpm"}, + "sweet_xml": {:hex, :sweet_xml, "0.6.5", "dd9cde443212b505d1b5f9758feb2000e66a14d3c449f04c572f3048c66e6697", [:mix], [], "hexpm"}, + "unicode_util_compat": {:hex, :unicode_util_compat, "0.2.0", "dbbccf6781821b1c0701845eaf966c9b6d83d7c3bfc65ca2b78b88b8678bfa35", [:rebar3], [], "hexpm"}} diff --git a/test/exfile_s3/backend_test.exs b/test/exfile_s3/backend_test.exs new file mode 100644 index 0000000..fd7f347 --- /dev/null +++ b/test/exfile_s3/backend_test.exs @@ -0,0 +1,16 @@ +defmodule ExfileS3.BackendTest do + use Exfile.BackendTest, { + ExfileS3.Backend, name: "test" + } + + alias Exfile.LocalFile + + test "uploading a file through S3 backend works", %{backend: backend} do + string = "hello from S3" + {:ok, file} = upload_string(backend, string) + {:ok, file2} = Backend.upload(backend, file) + {:ok, s3_file} = Backend.open(backend, file2.id) + + assert IO.binread(s3_file.io, :all) == string + end +end