Add first-seen and last-seen DNS record tracking

dooked/include/utils/containers.hpp

@@ -45,9 +45,17 @@ template <typename ValueType> class map_container_t {
 
   void append_impl(std::string const &key, ValueType const &value) {
     auto &container = map_[key].dns_result_list_;
-    auto iter = std::find(container.cbegin(), container.cend(), value);
-    if (iter == container.cend()) {
-      container.push_back(value);
+    auto iter = std::find(container.begin(), container.end(), value);
+    if (iter != container.end()) {
+      iter->last_seen = ValueType::now_epoch();
+      ++iter->seen_count;
+    } else {
+      auto rec = value;
+      auto const ts = ValueType::now_epoch();
+      rec.first_seen = ts;
+      rec.last_seen = ts;
+      rec.seen_count = 1;
+      container.push_back(std::move(rec));
     }
   }
 

dooked/include/utils/io_utils.hpp

@@ -8,13 +8,29 @@
 #include <nlohmann/json.hpp>
 #include <optional>
 #include <sstream>
+#include <cstdint>
+#include <type_traits>
 
 namespace dooked {
 
 using json = nlohmann::json;
 template <typename T> using opt_list_t = std::optional<std::vector<T>>;
 
+template <typename T, typename = void>
+struct has_tracking_fields : std::false_type {};
+
+template <typename T>
+struct has_tracking_fields<T, std::void_t<
+    decltype(std::declval<T>().first_seen),
+    decltype(std::declval<T>().last_seen),
+    decltype(std::declval<T>().seen_count)
+>> : std::true_type {};
+
+template <typename T>
+inline constexpr bool has_tracking_fields_v = has_tracking_fields<T>::value;
+
 void to_json(json &j, probe_result_t const &record);
+void from_json(json const &j, probe_result_t &record);
 dns_record_type_e dns_str_to_record_type(std::string const &);
 bool is_text_file(std::string const &file_extension);
 bool is_json_file(std::string const &file_extension);
@@ -26,9 +42,12 @@ void trim(std::string &);
 struct json_data_t {
   std::string domain_name{};
   std::string rdata{};
+  std::uint64_t first_seen{};
+  std::uint64_t last_seen{};
   int ttl{};
   int http_code{};
   int content_length{};
+  std::uint32_t seen_count{};
   dns_record_type_e type{};
 
   static json_data_t serialize(std::string const &d, int const len,
@@ -42,6 +61,18 @@ struct json_data_t {
     data.ttl = json_object["ttl"].get<json::number_integer_t>();
     data.content_length = len;
     data.http_code = http_code;
+    auto const it_fs = json_object.find("first-seen");
+    if (it_fs != json_object.end() && it_fs->second.is_number()) {
+      data.first_seen = it_fs->second.get<std::uint64_t>();
+    }
+    auto const it_ls = json_object.find("last-seen");
+    if (it_ls != json_object.end() && it_ls->second.is_number()) {
+      data.last_seen = it_ls->second.get<std::uint64_t>();
+    }
+    auto const it_sc = json_object.find("seen");
+    if (it_sc != json_object.end() && it_sc->second.is_number()) {
+      data.seen_count = it_sc->second.get<std::uint32_t>();
+    }
     return data;
   }
 };

dooked/include/utils/probe_result.hpp

@@ -1,6 +1,8 @@
 #pragma once
 
 #include "utils/constants.hpp"
+#include <chrono>
+#include <cstdint>
 #include <string>
 
 namespace dooked {
@@ -12,6 +14,16 @@ struct probe_result_t {
   std::string rdata{};
   dns_record_type_e type{}; // RR TYPE (2 octets)
   std::uint32_t ttl{};      // time to live(4 octets)
+  std::uint64_t first_seen{}; // epoch seconds, set on first observation
+  std::uint64_t last_seen{};  // epoch seconds, updated on every observation
+  std::uint32_t seen_count{}; // number of times this record was observed
+
+  static std::uint64_t now_epoch() {
+    return static_cast<std::uint64_t>(
+        std::chrono::duration_cast<std::chrono::seconds>(
+            std::chrono::system_clock::now().time_since_epoch())
+            .count());
+  }
 
   friend bool operator==(probe_result_t const &a, probe_result_t const &b) {
     return case_insensitive_compare(a.rdata, b.rdata) && (a.type == b.type);

dooked/source/cli_preprocessor.cpp

@@ -12,6 +12,8 @@
 // defined (and assigned to) in main.cpp
 extern bool silent;
 extern bool compare_cl;
+extern bool flag_first_seen;
+extern bool flag_last_seen;
 
 namespace dooked {
 
@@ -74,9 +76,12 @@ void compare_http_result(int const base_cl, json_data_t const &prev_http_result,
                    case_insensitive_compare(a.rdata, b.rdata);
           });
       if (!found) {
-        spdlog::error("[MISSING][{}][{}] `{}`", iter->domain_name,
-                      dns_record_type_to_str(start_iter->type),
-                      start_iter->rdata);
+        bool const any_filter = flag_first_seen || flag_last_seen;
+        if (!any_filter || flag_last_seen) {
+          spdlog::error("[MISSING][{}][{}] `{}`", iter->domain_name,
+                        dns_record_type_to_str(start_iter->type),
+                        start_iter->rdata);
+        }
       }
     }
     // information may have been changed
@@ -98,21 +103,27 @@ void compare_http_result(int const base_cl, json_data_t const &prev_http_result,
       if (find_iter == eq_range.second) {
         auto const distance = std::distance(eq_range.first, eq_range.second);
         if (distance == 0) {
-          spdlog::error("[REMOVED][{}][{}] `{}`", iter->domain_name,
-                        dns_record_type_to_str(start_iter->type),
-                        start_iter->rdata);
+          bool const any_filter = flag_first_seen || flag_last_seen;
+          if (!any_filter || flag_last_seen) {
+            spdlog::error("[REMOVED][{}][{}] `{}`", iter->domain_name,
+                          dns_record_type_to_str(start_iter->type),
+                          start_iter->rdata);
+          }
         } else if (distance == 1) {
           spdlog::info("[CHANGED][{}][{}] from `{}` to `{}`", iter->domain_name,
                        dns_record_type_to_str(start_iter->type),
                        start_iter->rdata, eq_range.first->rdata);
         } else {
           if (record_type != iter->type) {
             record_type = iter->type;
-            for (auto current_range = eq_range.first;
-                 current_range != eq_range.second; ++current_range) {
-              spdlog::info("[NEW][{}][{}] `{}`", iter->domain_name,
-                           dns_record_type_to_str(current_range->type),
-                           current_range->rdata);
+            bool const any_filter = flag_first_seen || flag_last_seen;
+            if (!any_filter || flag_first_seen) {
+              for (auto current_range = eq_range.first;
+                   current_range != eq_range.second; ++current_range) {
+                spdlog::info("[NEW][{}][{}] `{}`", iter->domain_name,
+                             dns_record_type_to_str(current_range->type),
+                             current_range->rdata);
+              }
             }
           }
         }
@@ -127,9 +138,12 @@ void compare_http_result(int const base_cl, json_data_t const &prev_http_result,
                    case_insensitive_compare(a.rdata, b.rdata);
           });
       if (!found) {
-        spdlog::info("[NEW][{}][{}] `{}`", iter->domain_name,
-                     dns_record_type_to_str(current_elem.type),
-                     current_elem.rdata);
+        bool const any_filter = flag_first_seen || flag_last_seen;
+        if (!any_filter || flag_first_seen) {
+          spdlog::info("[NEW][{}][{}] `{}`", iter->domain_name,
+                       dns_record_type_to_str(current_elem.type),
+                       current_elem.rdata);
+        }
       }
     }
   }
@@ -288,6 +302,26 @@ bool read_input_file(cli_args_t const &cli_args, runtime_args_t &rt_args) {
   return false;
 }
 
+void merge_tracking_fields(map_container_t<probe_result_t> &result_map,
+                           std::vector<json_data_t> const &previous_data) {
+  for (auto &[domain, response] : result_map.result()) {
+    for (auto &record : response.dns_result_list_) {
+      for (auto const &prev : previous_data) {
+        if (prev.domain_name == domain && prev.type == record.type &&
+            case_insensitive_compare(prev.rdata, record.rdata)) {
+          if (prev.first_seen != 0) {
+            record.first_seen = prev.first_seen;
+          }
+          if (prev.seen_count != 0) {
+            record.seen_count += prev.seen_count;
+          }
+          break;
+        }
+      }
+    }
+  }
+}
+
 void start_name_checking(runtime_args_t &&rt_args) {
   std::size_t const user_specified_thread =
       (rt_args.thread_count > 0) ? (std::size_t)rt_args.thread_count
@@ -350,6 +384,9 @@ void start_name_checking(runtime_args_t &&rt_args) {
     }
     thread_pool->join();
   }
+  if (rt_args.previous_data) {
+    merge_tracking_fields(result_map, *rt_args.previous_data);
+  }
   if (!silent) {
     spdlog::info("Writing JSON output");
   }

dooked/source/main.cpp

@@ -9,6 +9,8 @@
 bool no_bytes_count = false;
 bool silent = false;
 bool compare_cl = false;
+bool flag_first_seen = false;
+bool flag_last_seen = false;
 
 int main(int argc, char **argv) {
   CLI::App app{"dooked -- a CLI tool to enumerate DNS info"};
@@ -47,6 +49,10 @@ int main(int argc, char **argv) {
                "program returns 0 as the content-length as opposed the total"
                "bytes returned from the call to I/O socket read");
   app.add_flag("--silent", silent, "do not show any DNS or HTTP info");
+  app.add_flag("--fs", flag_first_seen,
+               "show only first-seen (new) DNS records; suppresses last-seen");
+  app.add_flag("--ls", flag_last_seen,
+               "show only last-seen (missing/removed) DNS records; suppresses first-seen");
 
   CLI11_PARSE(app, argc, argv);
 

dooked/source/utils/io_utils.cpp

@@ -6,6 +6,22 @@ void to_json(json &j, probe_result_t const &record) {
   j = json{{"ttl", record.ttl},
            {"type", dns_record_type_to_str(record.type)},
            {"info", record.rdata}};
+  if constexpr (has_tracking_fields_v<probe_result_t>) {
+    j["first-seen"] = record.first_seen;
+    j["last-seen"] = record.last_seen;
+    j["seen"] = record.seen_count;
+  }
+}
+
+void from_json(json const &j, probe_result_t &record) {
+  record.ttl = j.value("ttl", std::uint32_t{0});
+  record.type = dns_str_to_record_type(j.value("type", std::string{}));
+  record.rdata = j.value("info", std::string{});
+  if constexpr (has_tracking_fields_v<probe_result_t>) {
+    record.first_seen = j.value("first-seen", std::uint64_t{0});
+    record.last_seen = j.value("last-seen", std::uint64_t{0});
+    record.seen_count = j.value("seen", std::uint32_t{0});
+  }
 }
 
 bool is_text_file(std::string const &file_extension) {