Add tests; Update ex_aws

README.md

@@ -6,29 +6,45 @@ An AWS S3 adapter for [exfile](https://github.com/keichan34/exfile), using the [
 
 If [available in Hex](https://hex.pm/docs/publish), the package can be installed as:
 
-  1. Add exfile_s3 to your list of dependencies in `mix.exs`:
+  1. Add `exfile_s3` to your list of dependencies in `mix.exs`:
 
+      ```elixir
         def deps do
-          [{:exfile_s3, "~> 0.0.1"}]
+          [{:exfile_s3, "~> 0.0.3"}]
         end
+      ```
+  2. Ensure `exfile_s3` is started before your application:
 
-  2. Ensure exfile_s3 is started before your application:
-
+      ```elixir
         def application do
           [applications: [:exfile_s3]]
         end
+      ```
+
+  3. Configure `ex_aws` in config.exs:
 
-  3. Configure the backend in config.exs (or environment equivalent)
+      ```elixir
+        config :ex_aws, :s3,
+          access_key_id: System.get_env("AWS_ACCESS_KEY_ID"),
+          secret_access_key: System.get_env("AWS_SECRET_ACCESS_KEY"),
+          region: System.get_env("AWS_REGION"),
+          bucket: System.get_env("AWS_S3_BUCKET")
+      ```
 
+  4. Add the `exfile` backend in config.exs (or environment equivalent)
+
+      ```elixir
         config :exfile, Exfile,
           backends: %{
-            "store": {ExfileS3.Backend,
+            "store" => {
+              ExfileS3.Backend,
+              hasher: Exfile.Hasher.Random,
+              cdn_host: "CDN Host"
+            },
+            "cache" => {
+              ExfileS3.Backend,
               hasher: Exfile.Hasher.Random,
-              access_key_id: "AWS Access Key Id",
-              secret_access_key: "AWS Secret Access Key",
-              s3_prefix: "bucket prefix",
-              bucket_region: "Bucket Region",
-              cdn_host: "CDN Host",
-              bucket: "Name of the bucket to store files"
+              cdn_host: "CDN Host"
             }
           }
+      ```

config/config.exs

@@ -27,4 +27,5 @@ use Mix.Config
 # Configuration from the imported file will override the ones defined
 # here (which is why it is important to import them last).
 #
-#     import_config "#{Mix.env}.exs"
+
+import_config "#{Mix.env}.exs"

config/test.exs

@@ -0,0 +1,7 @@
+use Mix.Config
+
+config :ex_aws, :s3,
+  access_key_id: System.get_env("AWS_ACCESS_KEY_ID"),
+  secret_access_key: System.get_env("AWS_SECRET_ACCESS_KEY"),
+  region: System.get_env("AWS_REGION"),
+  bucket: System.get_env("AWS_S3_BUCKET")

lib/exfile_s3/backend.ex

@@ -1,14 +1,13 @@
 defmodule ExfileS3.Backend do
   use Exfile.Backend
 
-  alias Exfile.LocalFile
-  alias ExfileS3.S3
-
   def init(opts) do
-    _access_key_id     = Keyword.get(opts, :access_key_id)     || raise(ArgumentError, message: "access_key_id is required.")
-    _secret_access_key = Keyword.get(opts, :secret_access_key) || raise(ArgumentError, message: "secret_access_key is required.")
-    _bucket            = Keyword.get(opts, :bucket)            || raise(ArgumentError, message: "bucket is required.")
     {:ok, backend} = super(opts)
+    ExfileS3.Config.get(backend, :access_key_id)     || raise(ArgumentError, message: "access_key_id is required.")
+    ExfileS3.Config.get(backend, :secret_access_key) || raise(ArgumentError, message: "secret_access_key is required.")
+    ExfileS3.Config.get(backend, :region)            || raise(ArgumentError, message: "region is required.")
+    ExfileS3.Config.get(backend, :bucket)            || raise(ArgumentError, message: "bucket is required.")
+
     backend
   end
 
@@ -21,11 +20,11 @@ defmodule ExfileS3.Backend do
     end
   end
 
-  def upload(backend, %LocalFile{} = uploadable) do
-    id = backend.hasher.hash(uploadable)
-    case LocalFile.open(uploadable) do
+  def upload(backend, %Exfile.LocalFile{} = uploadable) do
+    file_id = backend.hasher.hash(uploadable)
+    case Exfile.LocalFile.open(uploadable) do
       {:ok, io} ->
-        perform_upload(backend, id, io)
+        perform_upload(backend, file_id, io)
       {:error, reason} ->
         {:error, reason}
     end
@@ -36,27 +35,32 @@ defmodule ExfileS3.Backend do
       {:error, reason} ->
         {:error, reason}
       iodata ->
-        S3.put_object(S3.find_config_value(:bucket), path(backend, file_id), iodata)
+        ExfileS3.Client.put_object(backend, path(backend, file_id), iodata)
         {:ok, get(backend, file_id)}
     end
   end
 
   def open(backend, file_id) do
-    case S3.get_object(S3.find_config_value(:bucket), path(backend, file_id)) do
+    case ExfileS3.Client.get_object(backend, path(backend, file_id)) do
       {:ok, %{body: body}} ->
         {:ok, io} = File.open(body, [:ram, :binary, :read])
-        {:ok, %LocalFile{io: io}}
+        {:ok, %Exfile.LocalFile{io: io}}
       {:error, reason} ->
         {:error, reason}
     end
   end
 
   def delete(backend, file_id) do
-    S3.delete_object(S3.find_config_value(:bucket), path(backend, file_id))
+    case exists?(backend, file_id) do
+      true  ->
+        ExfileS3.Client.delete_object(backend, path(backend, file_id)) |> elem(0)
+      false ->
+        :ok
+    end
   end
 
   def size(backend, file_id) do
-    case S3.head_object(S3.find_config_value(:bucket), path(backend, file_id)) do
+    case ExfileS3.Client.head_object(backend, path(backend, file_id)) do
       {:ok, %{headers: headers}} ->
         {"Content-Length", size} = Enum.find(headers, fn({header_name, _}) -> header_name == "Content-Length" end)
         {size, _} = Integer.parse(size)
@@ -67,14 +71,14 @@ defmodule ExfileS3.Backend do
   end
 
   def exists?(backend, file_id) do
-    case S3.head_object(S3.find_config_value(:bucket), path(backend, file_id)) do
+    case ExfileS3.Client.head_object(backend, path(backend, file_id)) do
       {:ok, _} -> true
       _ -> false
     end
   end
 
-  def path(_backend, id) do
-    [S3.find_config_value(:prefix), id]
+  def path(backend, id) do
+    [backend.backend_name, id]
     |> Enum.reject(&is_empty?/1)
     |> Enum.join("/")
   end

lib/exfile_s3/client.ex

@@ -0,0 +1,32 @@
+defmodule ExfileS3.Client do
+
+  def get_object(backend, path) do
+    backend
+    |> get_bucket()
+    |> ExAws.S3.get_object(path)
+    |> ExAws.request
+  end
+
+  def put_object(backend, path, iodata) do
+    backend
+    |> get_bucket()
+    |> ExAws.S3.put_object(path, iodata)
+    |> ExAws.request
+  end
+
+  def delete_object(backend, path) do
+    backend
+    |> get_bucket()
+    |> ExAws.S3.delete_object(path)
+    |> ExAws.request
+  end
+
+  def head_object(backend, path) do
+    backend
+    |> get_bucket()
+    |> ExAws.S3.head_object(path)
+    |> ExAws.request
+  end
+
+  defp get_bucket(backend), do: ExfileS3.Config.get(backend, :bucket)
+end

lib/exfile_s3/config.ex

@@ -0,0 +1,8 @@
+defmodule ExfileS3.Config do
+  def get(_,     :access_key_id), do: s3_config() |> Keyword.get(:access_key_id)
+  def get(_, :secret_access_key), do: s3_config() |> Keyword.get(:secret_access_key)
+  def get(_,            :bucket), do: s3_config() |> Keyword.get(:bucket)
+  def get(_,            :region), do: s3_config() |> Keyword.get(:region)
+
+  defp s3_config, do: Application.get_env(:ex_aws, :s3)
+end

mix.exs

@@ -4,13 +4,13 @@ defmodule ExfileS3.Mixfile do
   def project do
     [
       app: :exfile_s3,
-      version: "0.0.2",
+      version: "0.0.3",
       elixir: "~> 1.2",
       build_embedded: Mix.env == :prod,
       start_permanent: Mix.env == :prod,
-      deps: deps,
-      description: description,
-      package: package
+      deps: deps(),
+      description: description(),
+      package: package()
     ]
   end
 
@@ -48,11 +48,12 @@ defmodule ExfileS3.Mixfile do
   # Type "mix help deps" for more examples and options
   defp deps do
     [
-      {:exfile, "~> 0.3"},
-      {:ex_aws, "~> 0.5"},
-      {:sweet_xml, "~> 0.6"},
-      {:httpoison, "~> 0.8"},
-      {:poison, "~> 1.5 or ~> 2.0"}
+      # {:exfile, "~> 0.3"},
+      {:exfile, git: "git@github.com:grabr/exfile.git", branch: "phx-1.3"},
+      {:ex_aws, "~> 0.5 or ~> 1.1", optional: true},
+      {:sweet_xml, "~> 0.6", optional: true},
+      {:httpoison, "~> 0.8", optional: true},
+      {:poison, "~> 1.5 or ~> 2.0 or ~> 3.1", optional: true}
     ]
   end
 end

mix.lock

@@ -1,12 +1,14 @@
-%{"certifi": {:hex, :certifi, "0.4.0", "a7966efb868b179023618d29a407548f70c52466bf1849b9e8ebd0e34b7ea11f", [:rebar3], []},
-  "ex_aws": {:hex, :ex_aws, "0.5.0", "6ca02f1e8fe8340aa2eee66d9f08efcd6ff1f9f4ef7264669d0756e0b3917218", [:mix], [{:jsx, "~> 2.5", [hex: :jsx, optional: true]}, {:poison, "~> 1.2 or ~> 2.0", [hex: :poison, optional: true]}, {:httpoison, "~> 0.8", [hex: :httpoison, optional: true]}, {:sweet_xml, "~> 0.5", [hex: :sweet_xml, optional: true]}]},
-  "exfile": {:hex, :exfile, "0.3.3", "0c092011e42d47be4bde65ba6edfc76c13ef54fa221a50a1a311e2301b3335d3", [:mix], [{:poison, "~> 1.5 or ~> 2.0", [hex: :poison, optional: true]}, {:plug, "~> 1.0", [hex: :plug, optional: false]}, {:phoenix_html, "~> 2.3", [hex: :phoenix_html, optional: true]}, {:phoenix, "~> 1.1.4", [hex: :phoenix, optional: true]}, {:ecto, "~> 1.0", [hex: :ecto, optional: true]}]},
-  "hackney": {:hex, :hackney, "1.6.1", "ddd22d42db2b50e6a155439c8811b8f6df61a4395de10509714ad2751c6da817", [:rebar3], [{:ssl_verify_fun, "1.1.0", [hex: :ssl_verify_fun, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:metrics, "1.0.1", [hex: :metrics, optional: false]}, {:idna, "1.2.0", [hex: :idna, optional: false]}, {:certifi, "0.4.0", [hex: :certifi, optional: false]}]},
-  "httpoison": {:hex, :httpoison, "0.9.0", "68187a2daddfabbe7ca8f7d75ef227f89f0e1507f7eecb67e4536b3c516faddb", [:mix], [{:hackney, "~> 1.6.0", [hex: :hackney, optional: false]}]},
-  "idna": {:hex, :idna, "1.2.0", "ac62ee99da068f43c50dc69acf700e03a62a348360126260e87f2b54eced86b2", [:rebar3], []},
-  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []},
-  "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []},
-  "plug": {:hex, :plug, "1.1.5", "de5645c18170415a72b18cc3d215c05321ddecac27a15acb923742156e98278b", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, optional: true]}]},
-  "poison": {:hex, :poison, "2.2.0", "4763b69a8a77bd77d26f477d196428b741261a761257ff1cf92753a0d4d24a63", [:mix], []},
-  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.0", "edee20847c42e379bf91261db474ffbe373f8acb56e9079acb6038d4e0bf414f", [:rebar, :make], []},
-  "sweet_xml": {:hex, :sweet_xml, "0.6.1", "a56f235171f35a32807ce44798dedc748ce249fca574674fecd29c1321cab0de", [:mix], []}}
+%{"certifi": {:hex, :certifi, "1.2.1", "c3904f192bd5284e5b13f20db3ceac9626e14eeacfbb492e19583cf0e37b22be", [:rebar3], [], "hexpm"},
+  "ex_aws": {:hex, :ex_aws, "1.1.3", "08f3905e75ff0bb84f3e99156662e1620ba63e55a337eed6ab91d38d4f8f5a10", [:mix], [{:configparser_ex, "~> 0.2.1", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.0.6", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"},
+  "exfile": {:git, "git@github.com:grabr/exfile.git", "c10ee279aa54df5c8251c2b40ff2cc22e7b45406", [branch: "phx-1.3"]},
+  "hackney": {:hex, :hackney, "1.8.6", "21a725db3569b3fb11a6af17d5c5f654052ce9624219f1317e8639183de4a423", [:rebar3], [{:certifi, "1.2.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.0.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
+  "httpoison": {:hex, :httpoison, "0.12.0", "8fc3d791c5afe6beb0093680c667dd4ce712a49d89c38c3fe1a43100dd76cf90", [:mix], [{:hackney, "~> 1.8.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
+  "idna": {:hex, :idna, "5.0.2", "ac203208ada855d95dc591a764b6e87259cb0e2a364218f215ad662daa8cd6b4", [:rebar3], [{:unicode_util_compat, "0.2.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
+  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},
+  "mime": {:hex, :mime, "1.1.0", "01c1d6f4083d8aa5c7b8c246ade95139620ef8effb009edde934e0ec3b28090a", [:mix], [], "hexpm"},
+  "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"},
+  "plug": {:hex, :plug, "1.3.5", "7503bfcd7091df2a9761ef8cecea666d1f2cc454cbbaf0afa0b6e259203b7031", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1", [hex: :cowboy, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm"},
+  "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"},
+  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], [], "hexpm"},
+  "sweet_xml": {:hex, :sweet_xml, "0.6.5", "dd9cde443212b505d1b5f9758feb2000e66a14d3c449f04c572f3048c66e6697", [:mix], [], "hexpm"},
+  "unicode_util_compat": {:hex, :unicode_util_compat, "0.2.0", "dbbccf6781821b1c0701845eaf966c9b6d83d7c3bfc65ca2b78b88b8678bfa35", [:rebar3], [], "hexpm"}}

test/exfile_s3/backend_test.exs

@@ -0,0 +1,16 @@
+defmodule ExfileS3.BackendTest do
+  use Exfile.BackendTest, {
+    ExfileS3.Backend, name: "test"
+  }
+
+  alias Exfile.LocalFile
+
+  test "uploading a file through S3 backend works", %{backend: backend} do
+    string         = "hello from S3"
+    {:ok, file}    = upload_string(backend, string)
+    {:ok, file2}   = Backend.upload(backend, file)
+    {:ok, s3_file} = Backend.open(backend, file2.id)
+
+    assert IO.binread(s3_file.io, :all) == string
+  end
+end