diff --git a/Cargo.lock b/Cargo.lock
index 15a4221e..417b1c99 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -95,6 +95,20 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
 
+[[package]]
+name = "assert_cmd"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2475b58cd94eb4f70159f4fd8844ba3b807532fe3131b3373fae060bbe30396"
+dependencies = [
+ "bstr",
+ "doc-comment",
+ "predicates",
+ "predicates-core",
+ "predicates-tree",
+ "wait-timeout",
+]
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -194,6 +208,17 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae"
 
+[[package]]
+name = "bstr"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d"
+dependencies = [
+ "lazy_static",
+ "memchr",
+ "regex-automata",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.6.1"
@@ -531,6 +556,12 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "difference"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
+
 [[package]]
 name = "digest"
 version = "0.7.6"
@@ -580,6 +611,12 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "doc-comment"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
+
 [[package]]
 name = "dtoa"
 version = "0.4.8"
@@ -657,6 +694,15 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
 
+[[package]]
+name = "float-cmp"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -1418,6 +1464,12 @@ version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
 
+[[package]]
+name = "normalize-line-endings"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
+
 [[package]]
 name = "ntapi"
 version = "0.3.6"
@@ -1523,6 +1575,7 @@ checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
 name = "oktaws"
 version = "0.11.1"
 dependencies = [
+ "assert_cmd",
  "base64 0.13.0",
  "dialoguer",
  "dirs 3.0.1",
@@ -1534,6 +1587,7 @@ dependencies = [
  "log",
  "openssl",
  "path_abs",
+ "predicates",
  "pretty_env_logger",
  "rayon",
  "regex 1.4.5",
@@ -1771,6 +1825,35 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
 
+[[package]]
+name = "predicates"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eeb433456c1a57cc93554dea3ce40b4c19c4057e41c55d4a0f3d84ea71c325aa"
+dependencies = [
+ "difference",
+ "float-cmp",
+ "normalize-line-endings",
+ "predicates-core",
+ "regex 1.4.5",
+]
+
+[[package]]
+name = "predicates-core"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451"
+
+[[package]]
+name = "predicates-tree"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2"
+dependencies = [
+ "predicates-core",
+ "treeline",
+]
+
 [[package]]
 name = "pretty_env_logger"
 version = "0.4.0"
@@ -2001,6 +2084,15 @@ dependencies = [
  "regex-syntax 0.6.23",
 ]
 
+[[package]]
+name = "regex-automata"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4"
+dependencies = [
+ "byteorder",
+]
+
 [[package]]
 name = "regex-syntax"
 version = "0.5.6"
@@ -2932,6 +3024,12 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "treeline"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41"
+
 [[package]]
 name = "try-lock"
 version = "0.2.3"
@@ -3061,6 +3159,15 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
 
+[[package]]
+name = "wait-timeout"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "walkdir"
 version = "2.3.2"
diff --git a/Cargo.toml b/Cargo.toml
index fe417380..7dad1f33 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -43,3 +43,5 @@ openssl = { version = '0.10', features = ["vendored"] }
 
 [dev-dependencies]
 tempfile = "3"
+assert_cmd = "1"
+predicates = "1"
\ No newline at end of file
diff --git a/src/config/mod.rs b/src/config/mod.rs
index 3e8dea56..e03490aa 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -37,11 +37,10 @@ fn organizations_from_dir(dir: &Path) -> impl Iterator<Item = Organization> {
         .into_iter()
         .filter_map(|r| r.ok())
         .filter(|e| e.file_type().is_file())
-        .map(|e| e.path().try_into())
-        .filter_map(|r| match r {
+        .filter_map(|e| match e.path().try_into() {
             Ok(organization) => Some(organization),
-            Err(e) => {
-                error!("{:?}", e);
+            Err(err) => {
+                warn!("Could not parse {:?} as an Organization ({:?})", e.path(), err);
                 None
             }
         })
@@ -53,3 +52,36 @@ fn default_profile_location() -> Result<PathBuf, Error> {
         None => bail!("The environment variable HOME must be set."),
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    
+    use assert_cmd::prelude::*;
+    use tempfile::tempdir;
+
+    use std::process::Command;
+    use std::fs::File;
+    use std::io::Write;
+
+    #[test]
+    fn test_bad_organization_in_dir() {
+        let tmp_dir = tempdir().unwrap();
+
+        let file_path = tmp_dir.path().join("my-bad-org.toml");
+
+        let mut tmp_file = File::create(file_path).unwrap();
+        writeln!(tmp_file, "Not parseable as toml").unwrap();
+
+        assert_eq!(organizations_from_dir(tmp_dir.path()).count(), 0);
+
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME")).unwrap();
+        let assert = cmd
+            .env("OKTAWS_HOME", tmp_dir.path())
+            .assert();
+        assert
+            .failure()
+            .code(1)
+            .stderr(predicates::str::is_match(r#" WARN  oktaws::config > Could not parse ".+/my-bad-org.toml" as an Organization \(Error \{ inner: ErrorInner \{ kind: Wanted \{ expected: "an equals", found: "an identifier" \}, line: Some\(0\), col: 4, at: Some\(4\), message: "", key: \[\] \} \}\)\nError: ErrorMessage \{ msg: "No organizations found called \*" }\n"#).unwrap());
+    }
+}