diff --git a/.diataxis b/.diataxis index ccae2c2..f32050d 100644 --- a/.diataxis +++ b/.diataxis @@ -1,7 +1,6 @@ { + "default": "docs", "readme": "docs/README.md", - "howtos": "docs/how-to", - "tutorials": "docs/tutorials", - "adr": "docs/adr" + "adr": "docs/adr", + "projects": "docs/_gtd" } - diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 384440c..8d148b0 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,6 +1,6 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2026-07-08 19:38:12 UTC using RuboCop version 1.76.2. +# on 2026-07-09 11:16:03 UTC using RuboCop version 1.76.2. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new @@ -14,27 +14,32 @@ Gemspec/DevelopmentDependencies: Exclude: - 'bolt_dynamic_inventory.gemspec' -# Offense count: 2 +# Offense count: 4 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes. Metrics/AbcSize: - Max: 20 + Max: 26 # Offense count: 1 # Configuration parameters: CountComments, CountAsOne. Metrics/ClassLength: - Max: 129 + Max: 195 -# Offense count: 1 +# Offense count: 2 # Configuration parameters: AllowedMethods, AllowedPatterns. Metrics/CyclomaticComplexity: - Max: 8 + Max: 9 -# Offense count: 3 +# Offense count: 5 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. Metrics/MethodLength: Max: 20 # Offense count: 1 +# Configuration parameters: AllowedMethods, AllowedPatterns. +Metrics/PerceivedComplexity: + Max: 9 + +# Offense count: 3 RSpec/AnyInstance: Exclude: - 'spec/bolt_dynamic_inventory_spec.rb' @@ -46,15 +51,15 @@ RSpec/ContextWording: Exclude: - 'spec/bolt_dynamic_inventory_spec.rb' -# Offense count: 10 +# Offense count: 13 # Configuration parameters: CountAsOne. RSpec/ExampleLength: Max: 26 -# Offense count: 7 +# Offense count: 9 # Configuration parameters: AllowSubject. RSpec/MultipleMemoizedHelpers: - Max: 8 + Max: 11 # Offense count: 3 # Configuration parameters: AllowedGroups. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index f7ec97e..8344974 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,73 +2,45 @@ ## Our Pledge -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of experience, -nationality, personal appearance, race, religion, or sexual identity and -orientation. +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards -Examples of behavior that contributes to creating a positive environment -include: +Examples of behavior that contributes to creating a positive environment include: -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members Examples of unacceptable behavior by participants include: -* The use of sexualized language or imagery and unwelcome sexual attention or -advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting +- The use of sexualized language or imagery and unwelcome sexual attention or advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at gavin.didrichsen@gmail.com. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at . All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at [http://contributor-covenant.org/version/1/4][version] +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ diff --git a/README.md b/README.md index 937fe0f..295749b 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,35 @@ A Bolt inventory generator for VMs on various providers including orbstack and v ### System Dependencies - **Ruby** (version 2.7 or later) -- **nmap** (required for VMPooler connectivity filtering). For example, `brew install nmap` -- **floaty** (required for VMPooler). For example, `gem install floaty`. +- **nmap** (required for VMPooler connectivity filtering). For example, `brew install nmap` +- **floaty** (required for VMPooler). For example, `gem install floaty`. - **bolt** ## Usage -This `bolt_dynamic_inventory` gem queries either VMPooler or Orbstack and generates a Bolt inventory. It can be used in 2 ways: +### Quick Start -- **as a gem**. For more information see [How to use as a gem](./docs/how_to/how_to_use_as_a_gem.md). -- **as a bolt dynamic inventory plugin**. For more information see [How to use as a bolt dynamic plugin](./docs/how_to/how_to_use_as_a_bolt_dynamic_plugin.md). +This `bolt_dynamic_inventory` gem queries either VMPooler or Orbstack and generates a Bolt inventory. It can be used in 2 ways: + +- **as a gem**. For more information see [How to use as a gem](./docs/how_to/how_to_use_as_a_gem.md). +- **as a bolt dynamic inventory plugin**. For more information see [How to use as a bolt dynamic plugin](./docs/how_to/how_to_use_as_a_bolt_dynamic_plugin.md). This repository also explains [How to create a basic dynamic inventory plugin](./docs/how_to/how_to_create_a_basic_bolt_inventory_plugin.md). -For a listing of various how-to guides and design decisions, see the [documentation](./docs/README.md). +## References + +For a complete listing of various how-to guides and design decisions, see the [documentation](./docs/README.md). The following list is a good place to start. + +**Getting Started** + +- [How to setup the environment](howto_how_to_setup_the_environment.md) +- [How to use as a gem](howto_how_to_use_as_a_gem.md) +- [How to use as a bolt dynamic plugin](howto_how_to_use_as_a_bolt_dynamic_plugin.md) +- [How to use the `role` fact](howto_how_to_use_the_role_fact.md) +- [How to create a basic dynamic inventory plugin](howto_how_to_create_a_basic_dynamic_inventory_plugin.md) + +**Provider Guides** + +- [How to create and remove orbstack VMs from the command-line](howto_how_to_create_and_remove_orbstack_vms_from_the_command_line.md) +- [How to setup windows credentials for vmpooler](howto_how_to_setup_windows_credentials_for_vmpooler.md) +- [How to Test VMPooler Inventory Features](howto_how_to_test_vmpooler_inventory_features.md) diff --git a/docs/README.md b/docs/README.md index 97a6078..2b1253f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,22 +2,6 @@ ## Description -## Usage - -## Appendix - -### HowTos - - - - - -### Tutorials - - - - - ### Design Decisions @@ -28,4 +12,20 @@ * [ADR-0005](adr/0005-add-role-fact-that-matches-the-group-name-making-puppet-switching-easier.md) - Add 'role' fact that matches the group name making puppet switching easier * [ADR-0006](adr/0006-extend-the-plugin-to-handle-not-only-orbstack-but-vmpooler-as-well.md) - Extend the plugin to handle not only orbstack but vmpooler as well * [ADR-0007](adr/0007-use-nmap-ssh-port-scan-for-vmpooler-vm-connectivity-filtering.md) - Use nmap SSH port scan for VMPooler VM connectivity filtering +* [ADR-0008](adr/0008-query-both-abs-and-vmpooler-services-to-find-all-active-vms.md) - Query both ABS and vmpooler services to find all active VMs +* [ADR-0009](adr/0009-cache-vm-data-with-smart-ttl-based-invalidation.md) - Cache VM data with smart TTL-based invalidation +* [ADR-0010](adr/0010-surface-unavailable-vms-in-a-dedicated-group-instead-of-silently-dropping.md) - Surface unavailable VMs in a dedicated group instead of silently dropping + +### How-To Guides + + +* [How to create a basic dynamic inventory plugin](howto_how_to_create_a_basic_dynamic_inventory_plugin.md) +* [How to create and remove orbstack VMs from the command-line](howto_how_to_create_and_remove_orbstack_vms_from_the_command_line.md) +* [How to setup the environment](howto_how_to_setup_the_environment.md) +* [How to setup windows credentials for vmpooler](howto_how_to_setup_windows_credentials_for_vmpooler.md) +* [How to Test VMPooler Inventory Features](howto_how_to_test_vmpooler_inventory_features.md) +* [How to use as a bolt dynamic plugin](howto_how_to_use_as_a_bolt_dynamic_plugin.md) +* [How to use as a gem](howto_how_to_use_as_a_gem.md) +* [How to use the `role` fact](howto_how_to_use_the_role_fact.md) + diff --git a/docs/adr/0001-extend-this-gem-to-be-a-bolt-inventory-dynamic-plugin-also.md b/docs/adr/0001-extend-this-gem-to-be-a-bolt-inventory-dynamic-plugin-also.md index 6c1a420..fb281c0 100644 --- a/docs/adr/0001-extend-this-gem-to-be-a-bolt-inventory-dynamic-plugin-also.md +++ b/docs/adr/0001-extend-this-gem-to-be-a-bolt-inventory-dynamic-plugin-also.md @@ -8,7 +8,7 @@ Accepted ## Context -Bolt plugins are very useful for enhancing bolt functionality, particularly around inventory control. For example, a number of existing plugins exist to create dynamic inventory for Azure or AWS. These kinds of plugins are known as [reference plugins](https://www.puppet.com/docs/bolt/latest/writing_plugins#reference-plugins). While the implementation of a dynamic reference inventory plugin could live directly in the `tasks/resolve_reference.rb`, this is not good practice as it makes it difficult to test the logic. Also, it hides the underlying bolt `inventory.yaml` that is generated making it very difficult to troubleshoot. Further, this code is useful not only for a bolt dynamic inventory plugin but also for command-line. I might just want to generate a raw `inventory.yaml` for my orbstack and not use the dynamic plug in feature. +Bolt plugins are very useful for enhancing bolt functionality, particularly around inventory control. For example, a number of existing plugins exist to create dynamic inventory for Azure or AWS. These kinds of plugins are known as [reference plugins](https://www.puppet.com/docs/bolt/latest/writing_plugins#reference-plugins). While the implementation of a dynamic reference inventory plugin could live directly in the `tasks/resolve_reference.rb`, this is not good practice as it makes it difficult to test the logic. Also, it hides the underlying bolt `inventory.yaml` that is generated making it very difficult to troubleshoot. Further, this code is useful not only for a bolt dynamic inventory plugin but also for command-line. I might just want to generate a raw `inventory.yaml` for my orbstack and not use the dynamic plug in feature. One way to make the above possible is to make this code not only a gem for use in pure ruby, but also capable of acting as a bolt reference dynamic plugin. diff --git a/docs/adr/0002-configure-bolt-inventory-with-native-ssh-to-keep-things-simple.md b/docs/adr/0002-configure-bolt-inventory-with-native-ssh-to-keep-things-simple.md index bb9f366..66dc812 100644 --- a/docs/adr/0002-configure-bolt-inventory-with-native-ssh-to-keep-things-simple.md +++ b/docs/adr/0002-configure-bolt-inventory-with-native-ssh-to-keep-things-simple.md @@ -8,7 +8,7 @@ Accepted ## Context -Although there may be a number of ways to configure the SSH connectivity for a bolt dynamic inventory plugin, I want to implement this plugin as simply as possible and at the same time use the same ssh approach as on my terminal. Troubleshooting is easier: if my command-line cannot ssh to my orbstack VMs, then my plugin won't either. Fix the command-line and the plugin should work as well. +Although there may be a number of ways to configure the SSH connectivity for a bolt dynamic inventory plugin, I want to implement this plugin as simply as possible and at the same time use the same ssh approach as on my terminal. Troubleshooting is easier: if my command-line cannot ssh to my orbstack VMs, then my plugin won't either. Fix the command-line and the plugin should work as well. ## Decision diff --git a/docs/adr/0003-gather-inventory-metadata-via-the-cli-to-keep-things-simple.md b/docs/adr/0003-gather-inventory-metadata-via-the-cli-to-keep-things-simple.md index 4efa9b8..187d939 100644 --- a/docs/adr/0003-gather-inventory-metadata-via-the-cli-to-keep-things-simple.md +++ b/docs/adr/0003-gather-inventory-metadata-via-the-cli-to-keep-things-simple.md @@ -8,13 +8,13 @@ Accepted ## Context -When I discovered orbstack and in particular how easy and fast it was to create VMs, I immediately wanted to start using bolt to provision my orbstack VMs. Initially, I manually created the `inventory.yaml` but soon after created a ruby script to automatically generate the same `inventory.yaml`. Anytime I added or removed an orbstack VM I ran this script to generate an up-to-date bolt inventory. Since bolt plugins are useful for many tasks, including dynamic inventories, I decided to refactor my ruby script into a dynamic inventory plugin. One easy way to access the orbstack VM metadata is through the `orb` CLI. +When I discovered orbstack and in particular how easy and fast it was to create VMs, I immediately wanted to start using bolt to provision my orbstack VMs. Initially, I manually created the `inventory.yaml` but soon after created a ruby script to automatically generate the same `inventory.yaml`. Anytime I added or removed an orbstack VM I ran this script to generate an up-to-date bolt inventory. Since bolt plugins are useful for many tasks, including dynamic inventories, I decided to refactor my ruby script into a dynamic inventory plugin. One easy way to access the orbstack VM metadata is through the `orb` CLI. Other VM providers like vmpooler also have useful CLI commands for accessing VM metadata. ## Decision -Therefore, I decided to keep the implementation simple by using the CLI to access VM metadata. For orbstack VMs I use the `orb` cli; for vmpooler, the `floaty`. +Therefore, I decided to keep the implementation simple by using the CLI to access VM metadata. For orbstack VMs I use the `orb` cli; for vmpooler, the `floaty`. ## Consequences diff --git a/docs/adr/0004-create-dynamic-inventory-groups-based-on-hostname-regex-patterns.md b/docs/adr/0004-create-dynamic-inventory-groups-based-on-hostname-regex-patterns.md index 8667124..90d52f3 100644 --- a/docs/adr/0004-create-dynamic-inventory-groups-based-on-hostname-regex-patterns.md +++ b/docs/adr/0004-create-dynamic-inventory-groups-based-on-hostname-regex-patterns.md @@ -28,7 +28,7 @@ group_patterns: ## Consequences -The bolt_dynamic_inventory plugin now supports dynamic group creation based on target name patterns. This allows users to group their targets into logical groups based on their names. **NOTE**: +The bolt_dynamic_inventory plugin now supports dynamic group creation based on target name patterns. This allows users to group their targets into logical groups based on their names. **NOTE**: -* if `group_patterns` is empty, then no groups are created. -* if a group's regex pattern fails to match any targets, then that group is skipped. +- if `group_patterns` is empty, then no groups are created. +- if a group's regex pattern fails to match any targets, then that group is skipped. diff --git a/docs/adr/0005-add-role-fact-that-matches-the-group-name-making-puppet-switching-easier.md b/docs/adr/0005-add-role-fact-that-matches-the-group-name-making-puppet-switching-easier.md index 2bc5eab..68ade0b 100644 --- a/docs/adr/0005-add-role-fact-that-matches-the-group-name-making-puppet-switching-easier.md +++ b/docs/adr/0005-add-role-fact-that-matches-the-group-name-making-puppet-switching-easier.md @@ -8,11 +8,11 @@ Accepted ## Context -I want my bolt inventory.yaml to **not only** have 'groups' defined by regex, **but also** a 'role' fact defined for each of these groups. The role fact will have the same name as the group name. The reason this is useful is that bolt does not have to wait for puppet to collect facts on each target before switching on this 'role'. If I only wait for puppet to return a target's facts, then bolt is effectively blind for the first run. It cannot switch on particular target facts that it doesn't have. Since we're creating the bolt inventory and know relevant information about the targets, then I want this always available. +I want my bolt inventory.yaml to **not only** have 'groups' defined by regex, **but also** a 'role' fact defined for each of these groups. The role fact will have the same name as the group name. The reason this is useful is that bolt does not have to wait for puppet to collect facts on each target before switching on this 'role'. If I only wait for puppet to return a target's facts, then bolt is effectively blind for the first run. It cannot switch on particular target facts that it doesn't have. Since we're creating the bolt inventory and know relevant information about the targets, then I want this always available. ## Decision -Therefore, I decided to add a `${facts['role']}` for each group where the 'role' value is equal to the group name. In other words, given a group called `agent` then the `$facts.role = 'group'`. +Therefore, I decided to add a `${facts['role']}` for each group where the 'role' value is equal to the group name. In other words, given a group called `agent` then the `$facts.role = 'group'`. ## Consequences diff --git a/docs/adr/0006-extend-the-plugin-to-handle-not-only-orbstack-but-vmpooler-as-well.md b/docs/adr/0006-extend-the-plugin-to-handle-not-only-orbstack-but-vmpooler-as-well.md index 8c75bc5..3530340 100644 --- a/docs/adr/0006-extend-the-plugin-to-handle-not-only-orbstack-but-vmpooler-as-well.md +++ b/docs/adr/0006-extend-the-plugin-to-handle-not-only-orbstack-but-vmpooler-as-well.md @@ -18,21 +18,23 @@ Therefore, I decided to: 2. Create separate provider classes for Orbstack and VMPooler 3. Add a '--provider' command-line option to specify which provider to use 4. Implement VMPooler-specific features: - * Use `floaty list --active --json` for efficient VM discovery - * Add windows/linux group separation based on VM type - * Configure appropriate SSH settings for each group + - Use `floaty list --active --json` for efficient VM discovery + - Add windows/linux group separation based on VM type + - Configure appropriate SSH settings for each group 5. Maintain consistent features across providers: - * Dynamic group creation based on regex patterns - * Role facts that match group names - * Native SSH configuration + - Dynamic group creation based on regex patterns + - Role facts that match group names + - Native SSH configuration ## Consequences The plugin now supports both Orbstack and VMPooler VMs with a consistent interface. Benefits include: -* Users can manage both types of VMs using the same tool and configuration patterns -* Common features like regex-based grouping work identically across providers -* Each provider can implement its own optimal way of discovering and configuring VMs -* The provider abstraction makes it easy to add support for additional VM providers in the future +- Users can manage both types of VMs using the same tool and configuration patterns +- Common features like regex-based grouping work identically across providers +- Each provider can implement its own optimal way of discovering and configuring VMs +- The provider abstraction makes it easy to add support for additional VM providers in the future **NOTE**: The provider must be specified either in the inventory configuration or via the --provider command-line option. + +**Update (2026-07-09)**: The VMPooler provider now queries both ABS and vmpooler services to discover all active VMs (see [[0008-query-both-abs-and-vmpooler-services-to-find-all-active-vms|ADR-0008]]), caches results with smart TTL-based invalidation (see [[0009-cache-vm-data-with-smart-ttl-based-invalidation|ADR-0009]]), and surfaces unreachable VMs in an `unavailable` group (see [[0010-surface-unavailable-vms-in-a-dedicated-group-instead-of-silently-dropping|ADR-0010]]). diff --git a/docs/adr/0007-use-nmap-ssh-port-scan-for-vmpooler-vm-connectivity-filtering.md b/docs/adr/0007-use-nmap-ssh-port-scan-for-vmpooler-vm-connectivity-filtering.md index 4363db5..98d146c 100644 --- a/docs/adr/0007-use-nmap-ssh-port-scan-for-vmpooler-vm-connectivity-filtering.md +++ b/docs/adr/0007-use-nmap-ssh-port-scan-for-vmpooler-vm-connectivity-filtering.md @@ -32,7 +32,7 @@ We initially considered using RDP port 3389 for Windows detection, but found tha - **Fast parallel scanning** - nmap can check multiple hosts simultaneously - **Reliable Windows detection** - `-Pn` flag bypasses ping issues common with Windows -- **No additional API overhead** - Single nmap command replaces multiple floaty API calls +- **No additional API overhead** - Single nmap command replaces multiple floaty API calls - **Universal compatibility** - SSH port 22 works for both Linux and Windows VMs - **Graceful failure handling** - DNS resolution failures are handled cleanly diff --git a/docs/adr/0008-query-both-abs-and-vmpooler-services-to-find-all-active-vms.md b/docs/adr/0008-query-both-abs-and-vmpooler-services-to-find-all-active-vms.md new file mode 100644 index 0000000..d56fc85 --- /dev/null +++ b/docs/adr/0008-query-both-abs-and-vmpooler-services-to-find-all-active-vms.md @@ -0,0 +1,124 @@ + + +# 8. Query both ABS and vmpooler services to find all active VMs + +Date: 2026-07-09 + +## Status + +Accepted + +## Context + +The vmpooler provider originally used only `floaty list --active --json` (which queries the ABS service) to discover VMs. This missed VMs that were still running on vmpooler but no longer tracked by ABS. This was discovered when VMs known to be running did not appear in `binv` output. + +The root cause is that ABS and vmpooler have independent lifecycles. ABS tracks jobs by user — `floaty` fetches the entire ABS queue (`GET /api/v2/status/queue`) and filters client-side by `request.job.user`. When ABS reaps a job (its ABS lifecycle ends), the job disappears from the queue even though the underlying VMs may still be running on vmpooler (their vmpooler lifetime has not yet expired). Querying vmpooler directly (`GET /token/`) returns all VMs still alive under a given token, regardless of whether ABS still tracks them. This means a user's vmpooler VMs can outlive their ABS job records, making them invisible to an ABS-only query. + +## Decision + +Therefore, I decided to query both services and merge the results: + +- Query ABS via `floaty list --active --json` for VMs in `filled` or `allocated` state +- Query the vmpooler service via `floaty list --active --json --service vmpooler` for all VMs with their runtime metadata +- Merge the two lists by hostname: ABS VMs are enriched with `ttl` data from the vmpooler service when the same hostname appears in both; vmpooler-only VMs are appended to the list + +```ruby +# lib/bolt_dynamic_inventory/provider/vmpooler/inventory.rb — merge_vm_lists +def merge_vm_lists(abs_vms, vmpooler_vms) + vmpooler_by_host = vmpooler_vms.each_with_object({}) { |vm, h| h[vm['hostname']] = vm } + + merged = abs_vms.map do |abs_vm| + vmpooler_vm = vmpooler_by_host.delete(abs_vm['hostname']) + if vmpooler_vm && vmpooler_vm['ttl'] + abs_vm.merge('ttl' => vmpooler_vm['ttl']) + else + abs_vm + end + end + + merged.concat(vmpooler_by_host.values) +end +``` + +- ABS VMs are processed first; matching vmpooler entries are consumed (deleted from the lookup hash) to avoid duplicates +- The `ttl` field from vmpooler enriches ABS VMs so every target shows its reap time +- Any vmpooler VMs not matched to an ABS entry are appended — these are the previously-missing VMs + +## Consequences + +- All active VMs now appear in the Bolt inventory regardless of which service provisioned them +- The inventory output includes a `ttl` field per target showing when the VM will be reaped, calculated from vmpooler's `lifetime` and `running` fields +- Two `floaty` calls are made instead of one, adding a small amount of latency (mitigated by [[0009-cache-vm-data-with-smart-ttl-based-invalidation|ADR-0009]] caching) +- If either service is unavailable, the command fails — there is no graceful fallback to partial results + +## Related Topics + +- [[0006-extend-the-plugin-to-handle-not-only-orbstack-but-vmpooler-as-well|ADR-0006]]: Original vmpooler provider decision +- [[0003-gather-inventory-metadata-via-the-cli-to-keep-things-simple|ADR-0003]]: CLI-based metadata gathering approach +- [`e7d3e28`](https://github.com/gavindidrichsen/bolt_dynamic_inventory/commit/e7d3e28d14d4b418277810b9c231b7c61814a4bc): Commit implementing the dual-service query and merge diff --git a/docs/adr/0009-cache-vm-data-with-smart-ttl-based-invalidation.md b/docs/adr/0009-cache-vm-data-with-smart-ttl-based-invalidation.md new file mode 100644 index 0000000..368c4d9 --- /dev/null +++ b/docs/adr/0009-cache-vm-data-with-smart-ttl-based-invalidation.md @@ -0,0 +1,127 @@ + + +# 9. Cache VM data with smart TTL-based invalidation + +Date: 2026-07-09 + +## Status + +Accepted + +## Context + +Each `binv --provider=vmpooler` invocation makes two `floaty` API calls (ABS + vmpooler service, see [[0008-query-both-abs-and-vmpooler-services-to-find-all-active-vms|ADR-0008]]) plus an `nmap` scan of all returned VMs. This takes several seconds even for a small VM set. Since VMs are not frequently added or removed, repeating these queries on every invocation wastes time without improving accuracy. + +However, a simple time-based TTL cache is not sufficient. If a cached VM's TTL has already passed, the VM has likely been reaped by vmpooler and the cached inventory is stale. The cache must account for this. + +## Decision + +Therefore, I decided to add a file-based VM cache (`VmCache`) with three invalidation triggers: + +- **Manual refresh**: the `--refresh` / `-r` CLI flag bypasses the cache entirely +- **VM expiry**: if any cached VM's `ttl` timestamp has passed, the cache is invalidated (the VM was likely reaped) +- **Age-based TTL**: cache older than 24 hours is discarded regardless of VM state + +```ruby +# lib/bolt_dynamic_inventory/provider/vmpooler/inventory.rb — load_vms +def load_vms + unless @refresh + cached = @cache.read + return cached if cached && !@cache.any_vm_expired?(cached) + end + + vms = fetch_vmpooler_vms + @cache.write(vms) + vms +end +``` + +- `@refresh` skips the cache entirely when `--refresh` is passed +- `@cache.read` returns `nil` if the cache file is missing or older than 24 hours +- `any_vm_expired?` checks each VM's `ttl` against `Time.now` — if any VM has been reaped, we refetch + +Each provider gets its own directory under `~/.config/bolt_dynamic_inventory//`, containing: + +- `cache.json` — the raw VM data cache (JSON payload with `stored_at` timestamp, provider name, and full VM list) +- `inventory.yaml` — the generated Bolt inventory, written on every `generate` call so users can point Bolt at it directly with `bolt ... --inventoryfile ~/.config/bolt_dynamic_inventory/vmpooler/inventory.yaml` + +## Consequences + +- `binv` returns instantly on repeated calls within the cache window, making it practical for scripts and tab-completion workflows +- The persisted `inventory.yaml` makes it trivial to use the generated inventory directly with Bolt without piping `binv` output +- Accuracy is maintained: the cache self-invalidates when any VM's TTL expires, so reaped VMs never persist in the inventory +- The `--refresh` flag provides an escape hatch when the user knows the VM set has changed (e.g., immediately after provisioning) +- Each provider's data is isolated in its own directory under `~/.config/bolt_dynamic_inventory/` +- The 24-hour default TTL is a conservative fallback; most invalidation in practice comes from the VM expiry check + +## Related Topics + +- [[0008-query-both-abs-and-vmpooler-services-to-find-all-active-vms|ADR-0008]]: The dual-service query that caching wraps +- [[0007-use-nmap-ssh-port-scan-for-vmpooler-vm-connectivity-filtering|ADR-0007]]: Nmap scanning that caching also avoids repeating +- [`494a1ae`](https://github.com/gavindidrichsen/bolt_dynamic_inventory/commit/494a1ae00ad8dd526d20e0e21f52c310ab8cbb25): Commit implementing the VM cache diff --git a/docs/adr/0010-surface-unavailable-vms-in-a-dedicated-group-instead-of-silently-dropping.md b/docs/adr/0010-surface-unavailable-vms-in-a-dedicated-group-instead-of-silently-dropping.md new file mode 100644 index 0000000..08b3691 --- /dev/null +++ b/docs/adr/0010-surface-unavailable-vms-in-a-dedicated-group-instead-of-silently-dropping.md @@ -0,0 +1,120 @@ + + +# 10. Surface unavailable VMs in a dedicated group instead of silently dropping + +Date: 2026-07-09 + +## Status + +Accepted + +## Context + +The vmpooler provider uses nmap to check VM reachability (see [[0007-use-nmap-ssh-port-scan-for-vmpooler-vm-connectivity-filtering|ADR-0007]]). Previously, VMs that failed the nmap check were silently excluded from the inventory. This made it difficult to diagnose connectivity issues — a VM could be allocated and expected to be in the inventory, but its absence gave no indication of why. + +Silent drops are particularly confusing when VMs are still running but temporarily unreachable (network glitches, DNS propagation delays, or VMs still booting). The user sees fewer VMs than expected and has no signal about what happened. + +## Decision + +Therefore, I decided to place unreachable VMs into a dedicated `unavailable` group instead of dropping them: + +```ruby +# lib/bolt_dynamic_inventory/provider/vmpooler/inventory.rb — generate_inventory +def generate_inventory(alive_vms, unavailable_vms) + # ... targets and groups built from alive_vms ... + groups << unavailable_group(unavailable_names) unless unavailable_names.empty? + # ... +end + +def unavailable_group(targets) + { + 'name' => 'unavailable', + 'facts' => { 'role' => 'unavailable' }, + 'targets' => targets + } +end +``` + +- VMs failing nmap appear in the `unavailable` group with a `role: unavailable` fact +- The group is only added when there are actually unavailable VMs +- Unavailable VMs still appear in the top-level `targets` list with their full metadata (type, ttl) +- No transport config is set on the `unavailable` group — attempting to connect via Bolt will fail with a clear transport error rather than a silent omission + +## Consequences + +- Users can see at a glance which VMs are allocated but unreachable, making connectivity debugging straightforward +- Bolt plans can explicitly handle unavailable targets (e.g., skip them, wait and retry, or alert) +- The `unavailable` group name is reserved — it cannot be used as a regex group pattern name +- `binv` emits a yellow warning to stderr when unavailable VMs are present, so the user is immediately alerted without disrupting stdout (which remains valid YAML) +- The inventory output is slightly larger when VMs are unreachable, but this is a worthwhile trade for visibility + +## Related Topics + +- [[0007-use-nmap-ssh-port-scan-for-vmpooler-vm-connectivity-filtering|ADR-0007]]: The nmap scan that determines availability +- [[0005-add-role-fact-that-matches-the-group-name-making-puppet-switching-easier|ADR-0005]]: The `role` fact convention reused here diff --git a/docs/how_to/how_to_create_a_basic_dynamic_inventory_plugin.md b/docs/howto_how_to_create_a_basic_dynamic_inventory_plugin.md similarity index 88% rename from docs/how_to/how_to_create_a_basic_dynamic_inventory_plugin.md rename to docs/howto_how_to_create_a_basic_dynamic_inventory_plugin.md index 8ae1c46..9b3058c 100644 --- a/docs/how_to/how_to_create_a_basic_dynamic_inventory_plugin.md +++ b/docs/howto_how_to_create_a_basic_dynamic_inventory_plugin.md @@ -4,12 +4,12 @@ The following shows how to create a bare-bones bolt inventory plugin with only 6 files: -* `bolt-project.yaml​` -* `inventory.yaml​` -* `modules/basic_plugin/bolt-plugin.json​` -* `modules/basic_plugin/tasks/resolve_reference.json​` -* `modules/basic_plugin/tasks/resolve_reference.rb` -* `modules/basic_plugin/tasks/inventory.yaml` +- `bolt-project.yaml​` +- `inventory.yaml​` +- `modules/basic_plugin/bolt-plugin.json​` +- `modules/basic_plugin/tasks/resolve_reference.json​` +- `modules/basic_plugin/tasks/resolve_reference.rb` +- `modules/basic_plugin/tasks/inventory.yaml` ## Pre-requisites @@ -67,11 +67,11 @@ cat << 'EOL' > modules/basic_plugin/bolt_plugin.json EOL ``` -Create the `resolve_reference` task required by the plugin. In other words, +Create the `resolve_reference` task required by the plugin. In other words, -* create the `modules/basic_plugin/tasks/resolve_reference.json​` task metadata. -* create the `modules/basic_plugin/tasks/resolve_reference.rb` that loads the following hardcoded inventory.yaml -* create the `modules/basic_plugin/tasks/inventory.yaml` +- create the `modules/basic_plugin/tasks/resolve_reference.json​` task metadata. +- create the `modules/basic_plugin/tasks/resolve_reference.rb` that loads the following hardcoded inventory.yaml +- create the `modules/basic_plugin/tasks/inventory.yaml` ```bash # create the 'resolve_reference' task @@ -197,5 +197,5 @@ Finished on agent01: agent01 Successful on 5 targets: agent01,agent02,agent03,compiler01,compiler02 Ran on 5 targets in 11.03 sec -➜ my_plugin git:(development) +➜ my_plugin git:(development) ``` diff --git a/docs/how_to/how_to_create_and_remove_orbstack_vms_from_the_command_line.md b/docs/howto_how_to_create_and_remove_orbstack_vms_from_the_command_line.md similarity index 87% rename from docs/how_to/how_to_create_and_remove_orbstack_vms_from_the_command_line.md rename to docs/howto_how_to_create_and_remove_orbstack_vms_from_the_command_line.md index 8d632d9..93b1ca8 100644 --- a/docs/how_to/how_to_create_and_remove_orbstack_vms_from_the_command_line.md +++ b/docs/howto_how_to_create_and_remove_orbstack_vms_from_the_command_line.md @@ -8,14 +8,13 @@ The following shows how to both add and remove orbstack VMs from the command-lin Ensure the following are installed and configured: -* [orbstack](https://docs.orbstack.dev) +- [orbstack](https://docs.orbstack.dev) ## Usage ### Create orbstack VMs -Before proceeding, make sure to create 5 Ubuntu 22.04 amd64 orbstack machines in orbstack as follows `agent01`, `agent02`, `agent03`, `compiler01`, and `compiler02`. Do this manually or - +Before proceeding, make sure to create 5 Ubuntu 22.04 amd64 orbstack machines in orbstack as follows `agent01`, `agent02`, `agent03`, `compiler01`, and `compiler02`. Do this manually or ```bash # create 3 'agent0*' machines @@ -34,8 +33,7 @@ orbctl list ### Remove orbstack VMs -Before proceeding, make sure to create 5 Ubuntu 22.04 amd64 orbstack machines in orbstack as follows `agent01`, `agent02`, `agent03`, `compiler01`, and `compiler02`. Do this manually or - +Before proceeding, make sure to create 5 Ubuntu 22.04 amd64 orbstack machines in orbstack as follows `agent01`, `agent02`, `agent03`, `compiler01`, and `compiler02`. Do this manually or ```bash # create 3 'agent0*' machines diff --git a/docs/how_to/how_to_setup_the_environment.md b/docs/howto_how_to_setup_the_environment.md similarity index 78% rename from docs/how_to/how_to_setup_the_environment.md rename to docs/howto_how_to_setup_the_environment.md index e1c20ab..ea43b17 100644 --- a/docs/how_to/how_to_setup_the_environment.md +++ b/docs/howto_how_to_setup_the_environment.md @@ -8,13 +8,13 @@ Before doing any of the included `how-to's`, some or all of the following must b ### Orbstack Provider -* Install [orbstack](https://docs.orbstack.dev) -* Create Ubuntu 22.04 amd64 orbstack machines with names like: - * `agent01` - * `agent02` - * `agent03` - * `compiler01` - * `compiler02` +- Install [orbstack](https://docs.orbstack.dev) +- Create Ubuntu 22.04 amd64 orbstack machines with names like: + - `agent01` + - `agent02` + - `agent03` + - `compiler01` + - `compiler02` For more information on how to do the above from the command-line see [Create and Remove Orbstack VMs from the command-line](how_to_create_and_remove_orbstack_vms_from_cli.md). @@ -67,7 +67,7 @@ export VMPOOLER_WINDOWS_PASSWORD='' ### Install direnv -For more information see [direnv](https://direnv.net). This is a useful tool to automatically set environment variables on entry to a directory and then to unset on. For example: +For more information see [direnv](https://direnv.net). This is a useful tool to automatically set environment variables on entry to a directory and then to unset on. For example: Then create an `.envrc` file to configure the environment automatically via direnv: @@ -90,26 +90,26 @@ Install [rbenv](https://github.com/rbenv/rbenv), and then install a ruby version **NOTE**: If you already know how to configure bundler and have a preferred way of configuring your ruby environment, then skip this section. -Otherwise, isolate your ruby environment so that you don't accidentally corrupt your system. One way to do this isolation is by setting the following envionment variables: +Otherwise, isolate your ruby environment so that you don't accidentally corrupt your system. One way to do this isolation is by setting the following envionment variables: ```bash # isolate the ruby environment by setting the following environment variables for bundle and gem installations cat << 'EOL' > .envrc -# Configure Bundler paths -export BUNDLE_PATH="${PWD}/vendor/bundle" # Store gems locally -export BUNDLE_GEMFILE="${PWD}/Gemfile" # Use project-specific Gemfile -export BUNDLE_BIN="${PWD}/vendor/bin" # Store installed binaries +# Configure Bundler paths +export BUNDLE_PATH="${PWD}/vendor/bundle" # Store gems locally +export BUNDLE_GEMFILE="${PWD}/Gemfile" # Use project-specific Gemfile +export BUNDLE_BIN="${PWD}/vendor/bin" # Store installed binaries -# Configure RubyGems paths -export GEM_HOME="${PWD}/vendor/gems" # Local gem installation directory -export GEM_PATH="${PWD}/vendor/gems" # Lookup path for gems -export GEMRC="${PWD}/.gemrc" # Custom gem configuration +# Configure RubyGems paths +export GEM_HOME="${PWD}/vendor/gems" # Local gem installation directory +export GEM_PATH="${PWD}/vendor/gems" # Lookup path for gems +export GEMRC="${PWD}/.gemrc" # Custom gem configuration -# Update PATH to include local binaries -export PATH="${BUNDLE_BIN}:${GEM_HOME}/bin:$PATH" # Ensure executables are found +# Update PATH to include local binaries +export PATH="${BUNDLE_BIN}:${GEM_HOME}/bin:$PATH" # Ensure executables are found -# Suppress Bolt gem installation warning -export BOLT_GEM=true # Acknowledge Bolt is installed as a gem +# Suppress Bolt gem installation warning +export BOLT_GEM=true # Acknowledge Bolt is installed as a gem EOL diff --git a/docs/how_to/how_to_setup_windows_credentials_for_vmpooler.md b/docs/howto_how_to_setup_windows_credentials_for_vmpooler.md similarity index 79% rename from docs/how_to/how_to_setup_windows_credentials_for_vmpooler.md rename to docs/howto_how_to_setup_windows_credentials_for_vmpooler.md index 88d10a6..05449ab 100644 --- a/docs/how_to/how_to_setup_windows_credentials_for_vmpooler.md +++ b/docs/howto_how_to_setup_windows_credentials_for_vmpooler.md @@ -10,18 +10,18 @@ For bolt to connect to a vmpooler windows server over `winrm`, the `inventory.ya ```yaml groups: -- name: windows - config: - transport: winrm - winrm: - user: Administrator - password: - _plugin: env_var - var: VMPOOLER_WINDOWS_PASSWORD - ssl: false + - name: windows + config: + transport: winrm + winrm: + user: Administrator + password: + _plugin: env_var + var: VMPOOLER_WINDOWS_PASSWORD + ssl: false ``` -As long as the `VMPOOLER_WINDOWS_PASSWORD` environment variable is set to a valid password, then bolt will connect to the windows server(s). For more information, see the sample `inventory.yaml` produced by the `bolt_dynamic_inventory` gem in the [appendix](#sample-vmpoller-bolt-inventory). +As long as the `VMPOOLER_WINDOWS_PASSWORD` environment variable is set to a valid password, then bolt will connect to the windows server(s). For more information, see the sample `inventory.yaml` produced by the `bolt_dynamic_inventory` gem in the [appendix](#sample-vmpoller-bolt-inventory). ## Usage @@ -31,7 +31,7 @@ Export the following environment variable, making sure to replace ` # output a vmpooler inventory file -bundle exec binv --provider=vmpooler +bundle exec binv --provider=vmpooler # run a simple command via bolt, e.g., /opt/puppetlabs/bin/bolt command run ipconfig --verbose --inventoryfile=<(bundle exec binv --provider=vmpooler) --targets=windows @@ -42,7 +42,7 @@ bundle exec binv --provider=vmpooler ### Sample vmpoller bolt inventory ```bash -➜ bolt_dynamic_inventory git:(development) ✗ bundle exec binv --provider=vmpooler +➜ bolt_dynamic_inventory git:(development) ✗ bundle exec binv --provider=vmpooler --- targets: - name: onetime-algebra @@ -87,5 +87,5 @@ groups: targets: - tender-punditry - normal-meddling -➜ bolt_dynamic_inventory git:(development) ✗ +➜ bolt_dynamic_inventory git:(development) ✗ ``` diff --git a/docs/how_to/how_to_test_vmpooler_inventory_features.md b/docs/howto_how_to_test_vmpooler_inventory_features.md similarity index 61% rename from docs/how_to/how_to_test_vmpooler_inventory_features.md rename to docs/howto_how_to_test_vmpooler_inventory_features.md index 927786c..d19a1a9 100644 --- a/docs/how_to/how_to_test_vmpooler_inventory_features.md +++ b/docs/howto_how_to_test_vmpooler_inventory_features.md @@ -4,10 +4,10 @@ This guide shows you how to manually test the VMPooler inventory features to ens ## Prerequisites -* Access to VMPooler -* Bolt installed -* The bolt_dynamic_inventory module installed -* `nmap` installed (required for VM connectivity filtering) +- Access to VMPooler +- Bolt installed +- The bolt_dynamic_inventory module installed +- `nmap` installed (required for VM connectivity filtering) ## Test Scenarios @@ -23,9 +23,9 @@ binv generate --provider vmpooler Verify: -* Empty targets list -* Windows and Linux groups exist but have no targets -* Group configurations are present and correct +- Empty targets list +- Windows and Linux groups exist but have no targets +- Group configurations are present and correct ### 2. Basic Inventory Generation (With VMs) @@ -40,9 +40,9 @@ binv generate --provider vmpooler Verify: -* All VMs appear in targets list -* Windows VMs are in windows group with correct config -* Linux VMs are in linux group with correct config +- All VMs appear in targets list +- Windows VMs are in windows group with correct config +- Linux VMs are in linux group with correct config ### 3. Regex Group Pattern Testing @@ -57,9 +57,9 @@ binv generate --provider vmpooler --config '{"group_patterns": [{"group": "agent Verify: -* Base groups (windows/linux) exist and contain correct VMs -* 'agent' group exists and contains VMs matching the pattern -* Group facts and configurations are correct +- Base groups (windows/linux) exist and contain correct VMs +- 'agent' group exists and contains VMs matching the pattern +- Group facts and configurations are correct ### 4. VM Connectivity Filtering Testing @@ -83,10 +83,10 @@ binv generate --provider vmpooler Verify: -* Only reachable VMs appear in the inventory -* Destroyed VMs are automatically excluded from the targets -* The filtering happens automatically without manual intervention -* nmap connectivity checks work correctly for both Linux and Windows VMs +- Only reachable VMs appear in the inventory +- Destroyed VMs are automatically excluded from the targets +- The filtering happens automatically without manual intervention +- nmap connectivity checks work correctly for both Linux and Windows VMs **Note:** The connectivity filtering uses `nmap -Pn -p 22` to check SSH port availability, which works for both Linux and Windows VMPooler VMs. @@ -100,11 +100,11 @@ bundle exec floaty delete ## Troubleshooting -* If inventory generation fails, check VMPooler connectivity -* Verify VM hostnames in floaty output match expected patterns -* Check Windows credentials are properly configured if testing Windows VMs -* **nmap not found error**: Ensure `nmap` is installed on your system (see environment setup guide) -* **Connectivity filtering issues**: - * Verify nmap can reach VMPooler network (test with `nmap -Pn -p 22 `) - * Check firewall rules if VMs appear unreachable but should be accessible - * DNS resolution issues may cause VMs to be filtered out +- If inventory generation fails, check VMPooler connectivity +- Verify VM hostnames in floaty output match expected patterns +- Check Windows credentials are properly configured if testing Windows VMs +- **nmap not found error**: Ensure `nmap` is installed on your system (see environment setup guide) +- **Connectivity filtering issues**: + - Verify nmap can reach VMPooler network (test with `nmap -Pn -p 22 `) + - Check firewall rules if VMs appear unreachable but should be accessible + - DNS resolution issues may cause VMs to be filtered out diff --git a/docs/how_to/how_to_use_as_a_bolt_dynamic_plugin.md b/docs/howto_how_to_use_as_a_bolt_dynamic_plugin.md similarity index 93% rename from docs/how_to/how_to_use_as_a_bolt_dynamic_plugin.md rename to docs/howto_how_to_use_as_a_bolt_dynamic_plugin.md index 8fcef38..c889f70 100644 --- a/docs/how_to/how_to_use_as_a_bolt_dynamic_plugin.md +++ b/docs/howto_how_to_use_as_a_bolt_dynamic_plugin.md @@ -88,7 +88,7 @@ bolt inventory show --targets=windows bolt inventory show --targets=linux ``` -By adding a `group_patterns` section, then the bolt inventory will also include dynamic groups based on a regex pattern. For example, given a couple vmpooler VMs that begin with "tender" and "normal", then the following will include them in a new group called `agent`: +By adding a `group_patterns` section, then the bolt inventory will also include dynamic groups based on a regex pattern. For example, given a couple vmpooler VMs that begin with "tender" and "normal", then the following will include them in a new group called `agent`: ```bash # create a basic bolt inventory file that loads the plugin diff --git a/docs/how_to/how_to_use_as_a_gem.md b/docs/howto_how_to_use_as_a_gem.md similarity index 93% rename from docs/how_to/how_to_use_as_a_gem.md rename to docs/howto_how_to_use_as_a_gem.md index e063e8e..63988a4 100644 --- a/docs/how_to/how_to_use_as_a_gem.md +++ b/docs/howto_how_to_use_as_a_gem.md @@ -45,7 +45,7 @@ bolt command run "hostname" --inventoryfile=<(binv) --targets=all **COOL TIP**: Create a simple dynamic inventory with an alias, e.g., -If you've added the `bundle_dynamic_inventory` to your system viat a `gem install bundle_dynamic_inventory...`, then your should be able to use the `binv` command anywhere on your system. For example, I created the following aliases so that I could quickly run bolt commands against either vmpooler or orbstack: +If you've added the `bundle_dynamic_inventory` to your system viat a `gem install bundle_dynamic_inventory...`, then your should be able to use the `binv` command anywhere on your system. For example, I created the following aliases so that I could quickly run bolt commands against either vmpooler or orbstack: ```bash # create an alias that always runs the inventory @@ -88,7 +88,7 @@ bundle install Orbstack output: ```bash -➜ develop-the-bolt-dynamic-plugin git:(development) ✗ bundle exec binv --provider=orbstack +➜ develop-the-bolt-dynamic-plugin git:(development) ✗ bundle exec binv --provider=orbstack --- config: transport: ssh @@ -112,10 +112,10 @@ targets: uri: compiler01@orb - name: compiler02 uri: compiler02@orb -➜ develop-the-bolt-dynamic-plugin git:(development) ✗ +➜ develop-the-bolt-dynamic-plugin git:(development) ✗ ``` -* `bundle exec binv --groups "agent:agen*,compiler:comp*"` +- `bundle exec binv --groups "agent:agen*,compiler:comp*"` ```bash ➜ dump git:(development) ✗ bundle exec binv --provider=orbstack --groups "agent:agen*,compiler:comp*" @@ -209,5 +209,5 @@ groups: targets: - tender-punditry - normal-meddling -➜ bolt_dynamic_inventory git:(development) ✗ +➜ bolt_dynamic_inventory git:(development) ✗ ``` diff --git a/docs/how_to/how_to_use_the_role_fact.md b/docs/howto_how_to_use_the_role_fact.md similarity index 92% rename from docs/how_to/how_to_use_the_role_fact.md rename to docs/howto_how_to_use_the_role_fact.md index aeff318..90ef307 100644 --- a/docs/how_to/how_to_use_the_role_fact.md +++ b/docs/howto_how_to_use_the_role_fact.md @@ -2,7 +2,7 @@ ## Description -This document shows a simple use-case for the `role` fact, which is included in the dynamically generated bolt inventory. One advantage of this `role` fact is that bolt can use it on first execution **before** any facts have been collected by puppet on any of the targets. +This document shows a simple use-case for the `role` fact, which is included in the dynamically generated bolt inventory. One advantage of this `role` fact is that bolt can use it on first execution **before** any facts have been collected by puppet on any of the targets. The following example configures the `puppetlabs-motd` module using this fact. @@ -80,11 +80,11 @@ See sample output in the [appendix](#sample-output). ### Sample output -* run the plan injecting the `role` fact: +- run the plan injecting the `role` fact: ```bash # run the plan injecting the 'role' fact -➜ motd git:(development) ✗ /opt/puppetlabs/bin/bolt plan run usage::sayhello --targets=all --verbose +➜ motd git:(development) ✗ /opt/puppetlabs/bin/bolt plan run usage::sayhello --targets=all --verbose Starting: plan usage::sayhello Starting: install puppet and gather facts on agent01, agent02, agent03, compiler01, compiler02 @@ -113,10 +113,10 @@ Finished on agent02: Finished: apply catalog with 0 failures in 11.18 sec Finished: plan usage::sayhello in 29.5 sec Plan completed successfully with no result -➜ motd git:(development) ✗ +➜ motd git:(development) ✗ ``` -* verify the expected `/etc/motd` content: +- verify the expected `/etc/motd` content: ```bash # verify the expected content @@ -138,5 +138,5 @@ Finished on agent01: WELCOME! I'm an [agent] Successful on 5 targets: agent01,agent02,agent03,compiler01,compiler02 Ran on 5 targets in 1.85 sec -➜ motd git:(development) ✗ +➜ motd git:(development) ✗ ``` diff --git a/exe/binv b/exe/binv index fb0fa25..d66d24e 100755 --- a/exe/binv +++ b/exe/binv @@ -5,7 +5,7 @@ require 'bolt_dynamic_inventory' require 'yaml' require 'optparse' -options = { group_patterns: [], provider: 'orbstack' } +options = { group_patterns: [], provider: 'orbstack', refresh: false } # Parse command line options OptionParser.new do |opts| @@ -24,6 +24,10 @@ OptionParser.new do |opts| options[:provider] = provider end + opts.on('-r', '--refresh', 'Force cache refresh') do + options[:refresh] = true + end + opts.on('-v', '--version', 'Show version information') do puts "binv #{BoltDynamicInventory::VERSION}" exit @@ -32,10 +36,25 @@ end.parse! # Create an instance of your inventory class with the group patterns inventory = BoltDynamicInventory.new({ 'provider' => options[:provider], - 'group_patterns' => options[:group_patterns] }) + 'group_patterns' => options[:group_patterns], + 'refresh' => options[:refresh] }) # Generate the inventory inventory_data = inventory.generate +# Warn on stderr if there are unavailable VMs +if inventory_data['groups']&.any? { |g| g['name'] == 'unavailable' && !g['targets'].empty? } + unavailable = inventory_data['groups'].find { |g| g['name'] == 'unavailable' } + count = unavailable['targets'].length + suffix = count == 1 ? '' : 's' + warn "\e[33mWarning: #{count} unavailable VM#{suffix} moved to the 'unavailable' group — check connectivity\e[0m" +end + +# Report the persisted inventory path on stderr for providers that cache +if options[:provider] == 'vmpooler' + cache = BoltDynamicInventory::VmCache.new(provider: options[:provider]) + warn "Inventory written to #{cache.inventory_path}" +end + # Output as YAML puts inventory_data.to_yaml diff --git a/lib/bolt_dynamic_inventory/provider/vmpooler/inventory.rb b/lib/bolt_dynamic_inventory/provider/vmpooler/inventory.rb index 35aad0f..99a461a 100644 --- a/lib/bolt_dynamic_inventory/provider/vmpooler/inventory.rb +++ b/lib/bolt_dynamic_inventory/provider/vmpooler/inventory.rb @@ -3,7 +3,9 @@ require 'English' require 'json' require 'open3' +require 'set' require 'yaml' +require 'bolt_dynamic_inventory/vm_cache' module BoltDynamicInventory module Provider @@ -39,56 +41,70 @@ class Inventory def initialize(config = {}) @group_patterns = parse_group_patterns(config['group_patterns']) + @refresh = config['refresh'] || false + @cache = VmCache.new(provider: 'vmpooler') end - # Generate a Bolt inventory structure from VMPooler VMs def generate - vms = fetch_vmpooler_vms - vms = filter_alive_hosts(vms) - generate_inventory(vms) + vms = load_vms + alive, unavailable = partition_by_availability(vms) + inventory_data = generate_inventory(alive, unavailable) + @cache.write_inventory(inventory_data) + inventory_data end private + def load_vms + unless @refresh + cached = @cache.read + return cached if cached && !@cache.any_vm_expired?(cached) + end + + vms = fetch_vmpooler_vms + @cache.write(vms) + vms + end + def print_and_abort(message, stderr, status) error_msg = "#{message} with exit code #{status.exitstatus}" error_msg += ": #{stderr.strip}" unless stderr.empty? raise error_msg end - def filter_alive_hosts(vms) - return vms if vms.empty? + def partition_by_availability(vms) + return [[], []] if vms.empty? hostnames = vms.map { |item| item['hostname'] } - # Use -Pn to skip ping and check SSH port 22 - # This works for both Linux and Windows hosts in modern environments stdout, stderr, status = Open3.capture3('nmap', '-Pn', '-p', '22', *hostnames) print_and_abort('nmap failed', stderr, status) unless status.success? - # Extract hostnames from "Host is up" entries active_hostnames = stdout.lines .grep(/^Nmap scan report for/) - .map { |line| line.match(/^Nmap scan report for (\S+)/)[1] } - - return [] if active_hostnames.empty? + .to_set { |line| line.match(/^Nmap scan report for (\S+)/)[1] } - vms.select { |vm| active_hostnames.include?(vm['hostname']) } + alive = vms.select { |vm| active_hostnames.include?(vm['hostname']) } + unavailable = vms.reject { |vm| active_hostnames.include?(vm['hostname']) } + [alive, unavailable] end - # Fetch VMPooler VM details + # Fetch VMPooler VM details from both ABS and vmpooler services def fetch_vmpooler_vms - stdout, stderr, status = Open3.capture3('floaty list --active --json') + abs_vms = fetch_abs_vms + vmpooler_vms = fetch_vmpooler_service_vms + merge_vm_lists(abs_vms, vmpooler_vms) + end + + def fetch_abs_vms + stdout, stderr, status = Open3.capture3('floaty', 'list', '--active', '--json') print_and_abort('Failed to get VM list from floaty', stderr, status) unless status.success? - # Return empty array if stdout is empty (no VMs) return [] if stdout.strip.empty? - # Parse JSON stdout and return empty array if data is nil or empty data = JSON.parse(stdout) return [] if data.nil? || data.empty? - # Extract VMs that are in 'filled' or 'allocated' state and have allocated resources data.values.select { |job| %w[filled allocated].include?(job['state']) }.flat_map do |job| job['allocated_resources'].map do |resource| { @@ -99,26 +115,73 @@ def fetch_vmpooler_vms end end - # Generate the Bolt inventory structure - def generate_inventory(vms) - targets = extract_targets_with_type(vms) - target_names = targets.map { |t| t['name'] } - windows_targets, linux_targets = partition_targets_by_type(targets) + def fetch_vmpooler_service_vms + stdout, stderr, status = Open3.capture3('floaty', 'list', '--active', '--json', '--service', 'vmpooler') + print_and_abort('Failed to get VM list from vmpooler service', stderr, status) unless status.success? + + return [] if stdout.strip.empty? + + data = JSON.parse(stdout) + return [] if data.nil? || data.empty? + + now = Time.now + data.map do |short_name, vm_info| + vm = { + 'hostname' => vm_info['fqdn'] || "#{short_name}.delivery.puppetlabs.net", + 'type' => vm_info['template'] + } + lifetime = vm_info['lifetime'] + running = vm_info['running'] + if lifetime && running + remaining_hours = lifetime.to_f - running.to_f + reap_time = now + (remaining_hours * 3600) + vm['ttl'] = reap_time.strftime('%Y-%m-%d %H:%M') + end + vm + end + end + + def merge_vm_lists(abs_vms, vmpooler_vms) + vmpooler_by_host = vmpooler_vms.each_with_object({}) { |vm, h| h[vm['hostname']] = vm } + + merged = abs_vms.map do |abs_vm| + vmpooler_vm = vmpooler_by_host.delete(abs_vm['hostname']) + if vmpooler_vm && vmpooler_vm['ttl'] + abs_vm.merge('ttl' => vmpooler_vm['ttl']) + else + abs_vm + end + end + + merged.concat(vmpooler_by_host.values) + end + + def generate_inventory(alive_vms, unavailable_vms) + targets = extract_targets(alive_vms + unavailable_vms) + alive_names = alive_vms.map { |vm| vm['hostname'].split('.').first } + unavailable_names = unavailable_vms.map { |vm| vm['hostname'].split('.').first } + + alive_targets = targets.select { |t| alive_names.include?(t['name']) } + windows_targets, linux_targets = partition_targets_by_type(alive_targets) + + groups = base_groups(windows_targets, linux_targets) + groups << unavailable_group(unavailable_names) unless unavailable_names.empty? + groups.concat(regex_groups(alive_names)) { 'targets' => targets, - 'groups' => base_groups(windows_targets, linux_targets) + regex_groups(target_names) + 'groups' => groups } end - def extract_targets_with_type(vms) + def extract_targets(vms) vms.map do |vm| + vars = { 'type' => vm['type'] } + vars['ttl'] = vm['ttl'] if vm['ttl'] { 'name' => vm['hostname'].split('.').first, 'uri' => vm['hostname'], - 'vars' => { - 'type' => vm['type'] - } + 'vars' => vars } end end @@ -154,6 +217,14 @@ def linux_group(targets) } end + def unavailable_group(targets) + { + 'name' => 'unavailable', + 'facts' => { 'role' => 'unavailable' }, + 'targets' => targets + } + end + def regex_groups(target_names) @group_patterns.map do |pattern| { diff --git a/lib/bolt_dynamic_inventory/vm_cache.rb b/lib/bolt_dynamic_inventory/vm_cache.rb new file mode 100644 index 0000000..2477855 --- /dev/null +++ b/lib/bolt_dynamic_inventory/vm_cache.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +require 'json' +require 'fileutils' +require 'time' + +module BoltDynamicInventory + # TTL-based file cache for provider VM data. + class VmCache + DEFAULT_TTL_SECONDS = 86_400 # 24 hours + BASE_DIR = File.join(Dir.home, '.config', 'bolt_dynamic_inventory') + + def initialize(provider:, ttl: DEFAULT_TTL_SECONDS) + @provider = provider + @ttl = ttl + @provider_dir = File.join(BASE_DIR, provider) + end + + def read + return nil unless File.exist?(cache_path) + + data = JSON.parse(File.read(cache_path)) + expires_at = Time.parse(data['expires_at']) + return nil if Time.now >= expires_at + + data['vms'] + rescue JSON::ParserError, TypeError, ArgumentError + nil + end + + def write(vms) + FileUtils.mkdir_p(@provider_dir) + now = Time.now + effective_ttl = effective_ttl_for(vms, now) + expires_at = now + effective_ttl + payload = { + 'stored_at' => now.iso8601, + 'expires_at' => expires_at.iso8601, + 'expires_in' => humanize_duration(effective_ttl), + 'provider' => @provider, + 'vms' => vms + } + File.write(cache_path, JSON.pretty_generate(payload)) + end + + def write_inventory(inventory_data) + FileUtils.mkdir_p(@provider_dir) + header = inventory_header + File.write(inventory_path, header + inventory_data.to_yaml) + end + + def inventory_path + File.join(@provider_dir, 'inventory.yaml') + end + + def clear + FileUtils.rm_f(cache_path) + end + + def any_vm_expired?(vms) + now = Time.now + vms.any? do |vm| + next false unless vm['ttl'] + + reap_time = Time.parse(vm['ttl']) + reap_time <= now + end + rescue ArgumentError + false + end + + def stale? + !fresh? + end + + def fresh? + return false unless File.exist?(cache_path) + + data = JSON.parse(File.read(cache_path)) + expires_at = Time.parse(data['expires_at']) + Time.now < expires_at + rescue JSON::ParserError, TypeError, ArgumentError + false + end + + private + + def humanize_duration(seconds) + seconds = seconds.to_i + hours, remainder = seconds.divmod(3600) + minutes = remainder / 60 + parts = [] + parts << "#{hours} hour#{'s' unless hours == 1}" if hours.positive? + parts << "#{minutes} minute#{'s' unless minutes == 1}" if minutes.positive? + parts.empty? ? '0 minutes' : parts.join(', ') + end + + def inventory_header + return '' unless File.exist?(cache_path) + + data = JSON.parse(File.read(cache_path)) + stored = data['stored_at'] + expires = data['expires_at'] + "# created: #{stored}\n# expires: #{expires}\n" + rescue JSON::ParserError, TypeError + '' + end + + def effective_ttl_for(vms, now) + earliest_reap = earliest_vm_reap_time(vms) + return @ttl unless earliest_reap + + vm_remaining = (earliest_reap - now).to_i + return @ttl if vm_remaining <= 0 + + [vm_remaining, @ttl].min + end + + def earliest_vm_reap_time(vms) + reap_times = vms.filter_map do |vm| + Time.parse(vm['ttl']) if vm['ttl'] + rescue ArgumentError + nil + end + reap_times.min + end + + def cache_path + File.join(@provider_dir, 'cache.json') + end + end +end diff --git a/spec/bolt_dynamic_inventory/vm_cache_spec.rb b/spec/bolt_dynamic_inventory/vm_cache_spec.rb new file mode 100644 index 0000000..da7f688 --- /dev/null +++ b/spec/bolt_dynamic_inventory/vm_cache_spec.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'bolt_dynamic_inventory/vm_cache' +require 'tmpdir' +require 'json' + +RSpec.describe BoltDynamicInventory::VmCache do + let(:tmpdir) { Dir.mktmpdir } + let(:cache) { described_class.new(provider: 'test') } + + before do + stub_const('BoltDynamicInventory::VmCache::BASE_DIR', tmpdir) + end + + after do + FileUtils.remove_entry(tmpdir) + end + + describe '#write' do + it 'includes an expires_at field in the cache file' do + cache.write([{ 'hostname' => 'a.example.com', 'type' => 'ubuntu' }]) + data = JSON.parse(File.read(File.join(tmpdir, 'test', 'cache.json'))) + + expect(data).to have_key('expires_at') + expect(Time.parse(data['expires_at'])).to be > Time.now + end + + it 'caps expiry at the shortest-lived VM when shorter than default' do + now = Time.now + short_lived_reap = now + 3600 + vms = [ + { 'hostname' => 'a.example.com', 'type' => 'ubuntu', 'ttl' => short_lived_reap.strftime('%Y-%m-%d %H:%M') }, + { 'hostname' => 'b.example.com', 'type' => 'ubuntu', 'ttl' => (now + 86_400).strftime('%Y-%m-%d %H:%M') } + ] + + cache.write(vms) + data = JSON.parse(File.read(File.join(tmpdir, 'test', 'cache.json'))) + + expect(data).to have_key('expires_in') + expect(data).not_to have_key('ttl_seconds') + expires_at = Time.parse(data['expires_at']) + expect(expires_at).to be_within(60).of(short_lived_reap) + end + + it 'uses default TTL when no VMs have a ttl field' do + vms = [ + { 'hostname' => 'a.example.com', 'type' => 'ubuntu' }, + { 'hostname' => 'b.example.com', 'type' => 'ubuntu' } + ] + + cache.write(vms) + data = JSON.parse(File.read(File.join(tmpdir, 'test', 'cache.json'))) + + expect(data['expires_in']).to eq('24 hours') + end + + it 'uses default TTL when shortest VM TTL exceeds default' do + now = Time.now + vms = [ + { 'hostname' => 'a.example.com', 'type' => 'ubuntu', 'ttl' => (now + 100_000).strftime('%Y-%m-%d %H:%M') } + ] + + cache.write(vms) + data = JSON.parse(File.read(File.join(tmpdir, 'test', 'cache.json'))) + + expect(data['expires_in']).to eq('24 hours') + end + + it 'uses default TTL when VM ttl is already expired' do + past = Time.now - 3600 + vms = [ + { 'hostname' => 'a.example.com', 'type' => 'ubuntu', 'ttl' => past.strftime('%Y-%m-%d %H:%M') } + ] + + cache.write(vms) + data = JSON.parse(File.read(File.join(tmpdir, 'test', 'cache.json'))) + + expect(data['expires_in']).to eq('24 hours') + end + end + + describe '#write_inventory' do + it 'prepends created and expires comments from cache metadata' do + cache.write([{ 'hostname' => 'a.example.com', 'type' => 'ubuntu' }]) + inventory_data = { 'targets' => [], 'groups' => [] } + cache.write_inventory(inventory_data) + + content = File.read(cache.inventory_path) + expect(content).to match(/^# created: \d{4}-\d{2}-\d{2}/) + expect(content).to match(/^# expires: \d{4}-\d{2}-\d{2}/) + end + end + + describe '#read' do + it 'returns vms when cache has not expired' do + vms = [{ 'hostname' => 'a.example.com', 'type' => 'ubuntu' }] + cache.write(vms) + + expect(cache.read).to eq(vms) + end + + it 'returns nil when cache has expired' do + FileUtils.mkdir_p(File.join(tmpdir, 'test')) + payload = { + 'stored_at' => (Time.now - 200).iso8601, + 'expires_at' => (Time.now - 100).iso8601, + 'provider' => 'test', + 'expires_in' => '1 minute, 40 seconds', + 'vms' => [{ 'hostname' => 'a.example.com', 'type' => 'ubuntu' }] + } + File.write(File.join(tmpdir, 'test', 'cache.json'), JSON.pretty_generate(payload)) + + expect(cache.read).to be_nil + end + + it 'returns nil for old-format cache files without expires_at' do + FileUtils.mkdir_p(File.join(tmpdir, 'test')) + payload = { + 'stored_at' => Time.now.iso8601, + 'provider' => 'test', + 'expires_in' => '24 hours', + 'vms' => [{ 'hostname' => 'a.example.com', 'type' => 'ubuntu' }] + } + File.write(File.join(tmpdir, 'test', 'cache.json'), JSON.pretty_generate(payload)) + + expect(cache.read).to be_nil + end + end + + describe '#fresh?' do + it 'returns true when cache has not expired' do + cache.write([{ 'hostname' => 'a.example.com', 'type' => 'ubuntu' }]) + expect(cache.fresh?).to be true + end + + it 'returns false when cache has expired' do + FileUtils.mkdir_p(File.join(tmpdir, 'test')) + payload = { + 'stored_at' => (Time.now - 200).iso8601, + 'expires_at' => (Time.now - 100).iso8601, + 'provider' => 'test', + 'expires_in' => '1 minute, 40 seconds', + 'vms' => [] + } + File.write(File.join(tmpdir, 'test', 'cache.json'), JSON.pretty_generate(payload)) + + expect(cache.fresh?).to be false + end + + it 'returns false for old-format cache files' do + FileUtils.mkdir_p(File.join(tmpdir, 'test')) + payload = { + 'stored_at' => Time.now.iso8601, + 'provider' => 'test', + 'expires_in' => '24 hours', + 'vms' => [] + } + File.write(File.join(tmpdir, 'test', 'cache.json'), JSON.pretty_generate(payload)) + + expect(cache.fresh?).to be false + end + end +end diff --git a/spec/bolt_dynamic_inventory_spec.rb b/spec/bolt_dynamic_inventory_spec.rb index 05f2334..b69f98a 100644 --- a/spec/bolt_dynamic_inventory_spec.rb +++ b/spec/bolt_dynamic_inventory_spec.rb @@ -95,7 +95,13 @@ end describe BoltDynamicInventory::Provider::Vmpooler::Inventory do - let(:mock_vmpooler_json) do + before do + allow_any_instance_of(BoltDynamicInventory::VmCache).to receive(:read).and_return(nil) + allow_any_instance_of(BoltDynamicInventory::VmCache).to receive(:write) + allow_any_instance_of(BoltDynamicInventory::VmCache).to receive(:write_inventory) + end + + let(:mock_abs_json) do { 'job-1' => { 'state' => 'filled', @@ -113,6 +119,32 @@ }.to_json end + let(:mock_vmpooler_service_json) do + { + 'onetime-algebra' => { + 'template' => 'win-2019-x86_64', + 'fqdn' => 'onetime-algebra.delivery.puppetlabs.net', + 'state' => 'running', + 'lifetime' => 24, + 'running' => 3.5 + }, + 'tender-punditry' => { + 'template' => 'ubuntu-2004-x86_64', + 'fqdn' => 'tender-punditry.delivery.puppetlabs.net', + 'state' => 'running', + 'lifetime' => 24, + 'running' => 6.0 + }, + 'normal-meddling' => { + 'template' => 'ubuntu-2004-x86_64', + 'fqdn' => 'normal-meddling.delivery.puppetlabs.net', + 'state' => 'running', + 'lifetime' => 24, + 'running' => 6.0 + } + }.to_json + end + let(:mock_vms) do [ { 'hostname' => 'onetime-algebra.delivery.puppetlabs.net', 'type' => 'win-2019-x86_64' }, @@ -179,12 +211,24 @@ STDERR end + let(:success_status) { instance_double(Process::Status, success?: true) } + + def stub_abs_response(json_str) + allow(Open3).to receive(:capture3) + .with('floaty', 'list', '--active', '--json') + .and_return([json_str, '', success_status]) + end + + def stub_vmpooler_service_response(json_str) + allow(Open3).to receive(:capture3) + .with('floaty', 'list', '--active', '--json', '--service', 'vmpooler') + .and_return([json_str, '', success_status]) + end + context 'when no VMs are available' do before do - allow(Open3).to receive(:capture3) - .with('floaty list --active --json') - .and_return(['', '', instance_double(Process::Status, success?: true)]) - # No need to mock nmap since no VMs means no filtering needed + stub_abs_response('') + stub_vmpooler_service_response('') end it 'generates inventory with empty targets and base groups' do @@ -193,7 +237,6 @@ expect(result['targets']).to eq([]) - # Check windows group is empty but configured windows_group = result['groups'].find { |g| g['name'] == 'windows' } expect(windows_group['targets']).to eq([]) expect(windows_group['facts']).to eq('role' => 'windows') @@ -209,7 +252,6 @@ } ) - # Check linux group is empty but configured linux_group = result['groups'].find { |g| g['name'] == 'linux' } expect(linux_group['targets']).to eq([]) expect(linux_group['facts']).to eq('role' => 'linux') @@ -229,7 +271,6 @@ ) result = inventory.generate - # Check regex-based group is empty agent_group = result['groups'].find { |g| g['name'] == 'agent' } expect(agent_group['targets']).to eq([]) expect(agent_group['facts']).to eq('role' => 'agent') @@ -238,12 +279,11 @@ describe 'with available VMs' do before do - allow(Open3).to receive(:capture3) - .with('floaty list --active --json') - .and_return([mock_vmpooler_json, '', instance_double(Process::Status, success?: true)]) + stub_abs_response(mock_abs_json) + stub_vmpooler_service_response(mock_vmpooler_service_json) allow(Open3).to receive(:capture3) .with('nmap', '-Pn', '-p', '22', *hostnames) - .and_return([mock_nmap_output_all_alive, mock_nmap_stderr, instance_double(Process::Status, success?: true)]) + .and_return([mock_nmap_output_all_alive, mock_nmap_stderr, success_status]) end let(:inventory) { described_class.new } @@ -256,17 +296,16 @@ ) end - it 'includes vars with type for each target' do + it 'includes vars with type and ttl for each target' do targets = result['targets'] onetime_target = targets.find { |t| t['name'] == 'onetime-algebra' } - expect(onetime_target['vars']).to eq('type' => 'win-2019-x86_64') + expect(onetime_target['vars']['type']).to eq('win-2019-x86_64') + expect(onetime_target['vars']['ttl']).to match(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}$/) tender_target = targets.find { |t| t['name'] == 'tender-punditry' } - expect(tender_target['vars']).to eq('type' => 'ubuntu-2004-x86_64') - - normal_target = targets.find { |t| t['name'] == 'normal-meddling' } - expect(normal_target['vars']).to eq('type' => 'ubuntu-2004-x86_64') + expect(tender_target['vars']['type']).to eq('ubuntu-2004-x86_64') + expect(tender_target['vars']['ttl']).to match(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}$/) end it 'configures windows group correctly' do @@ -306,45 +345,189 @@ ) result = inventory.generate - # Base groups should still exist expect(result['groups'].find { |g| g['name'] == 'windows' }).not_to be_nil expect(result['groups'].find { |g| g['name'] == 'linux' }).not_to be_nil - # Check regex-based group agent_group = result['groups'].find { |g| g['name'] == 'agent' } expect(agent_group['targets']).to contain_exactly('tender-punditry', 'normal-meddling') expect(agent_group['facts']).to eq('role' => 'agent') end end - describe 'host filtering with nmap' do + describe 'merging VMs from ABS and vmpooler services' do + let(:extra_vmpooler_json) do + { + 'onetime-algebra' => { + 'template' => 'win-2019-x86_64', + 'fqdn' => 'onetime-algebra.delivery.puppetlabs.net', + 'state' => 'running', + 'lifetime' => 24, + 'running' => 3.5 + }, + 'tender-punditry' => { + 'template' => 'ubuntu-2004-x86_64', + 'fqdn' => 'tender-punditry.delivery.puppetlabs.net', + 'state' => 'running', + 'lifetime' => 24, + 'running' => 6.0 + }, + 'normal-meddling' => { + 'template' => 'ubuntu-2004-x86_64', + 'fqdn' => 'normal-meddling.delivery.puppetlabs.net', + 'state' => 'running', + 'lifetime' => 24, + 'running' => 6.0 + }, + 'tight-mantrap' => { + 'template' => 'ubuntu-2404-x86_64', + 'fqdn' => 'tight-mantrap.delivery.puppetlabs.net', + 'state' => 'running', + 'lifetime' => 24, + 'running' => 1.0 + }, + 'unhurt-gadgetry' => { + 'template' => 'ubuntu-2404-x86_64', + 'fqdn' => 'unhurt-gadgetry.delivery.puppetlabs.net', + 'state' => 'running', + 'lifetime' => 24, + 'running' => 1.0 + } + }.to_json + end + + let(:all_hostnames) do + %w[ + onetime-algebra.delivery.puppetlabs.net + tender-punditry.delivery.puppetlabs.net + normal-meddling.delivery.puppetlabs.net + tight-mantrap.delivery.puppetlabs.net + unhurt-gadgetry.delivery.puppetlabs.net + ] + end + + let(:mock_nmap_output_all_five) do + <<~NMAP + Starting Nmap 7.98 ( https://nmap.org ) at 2025-10-28 18:00 +0000 + Nmap scan report for onetime-algebra.delivery.puppetlabs.net (10.16.121.11) + Host is up (0.15s latency). + Nmap scan report for tender-punditry.delivery.puppetlabs.net (10.16.121.12) + Host is up (0.15s latency). + Nmap scan report for normal-meddling.delivery.puppetlabs.net (10.16.121.13) + Host is up (0.15s latency). + Nmap scan report for tight-mantrap.delivery.puppetlabs.net (10.16.121.14) + Host is up (0.15s latency). + Nmap scan report for unhurt-gadgetry.delivery.puppetlabs.net (10.16.121.15) + Host is up (0.15s latency). + Nmap done: 5 IP addresses (5 hosts up) scanned in 1.34 seconds + NMAP + end + + before do + stub_abs_response(mock_abs_json) + stub_vmpooler_service_response(extra_vmpooler_json) + allow(Open3).to receive(:capture3) + .with('nmap', '-Pn', '-p', '22', *all_hostnames) + .and_return([mock_nmap_output_all_five, '', success_status]) + end + + it 'includes VMs from both ABS and vmpooler services' do + inventory = described_class.new + result = inventory.generate + + expect(result['targets'].map { |t| t['name'] }).to contain_exactly( + 'onetime-algebra', 'tender-punditry', 'normal-meddling', + 'tight-mantrap', 'unhurt-gadgetry' + ) + end + + it 'does not duplicate VMs present in both sources' do + inventory = described_class.new + result = inventory.generate + + hostnames_in_result = result['targets'].map { |t| t['uri'] } + expect(hostnames_in_result.length).to eq(hostnames_in_result.uniq.length) + end + + it 'correctly groups merged VMs by OS type' do + inventory = described_class.new + result = inventory.generate + + windows_group = result['groups'].find { |g| g['name'] == 'windows' } + expect(windows_group['targets']).to contain_exactly('onetime-algebra') + + linux_group = result['groups'].find { |g| g['name'] == 'linux' } + expect(linux_group['targets']).to contain_exactly( + 'tender-punditry', 'normal-meddling', 'tight-mantrap', 'unhurt-gadgetry' + ) + end + end + + describe 'VMs only in vmpooler service' do + let(:vmpooler_only_json) do + { + 'tight-mantrap' => { + 'template' => 'ubuntu-2404-x86_64', + 'fqdn' => 'tight-mantrap.delivery.puppetlabs.net', + 'state' => 'running', + 'lifetime' => 24, + 'running' => 2.0 + } + }.to_json + end + + let(:vmpooler_only_nmap) do + <<~NMAP + Nmap scan report for tight-mantrap.delivery.puppetlabs.net (10.16.121.14) + Host is up (0.15s latency). + NMAP + end + before do + stub_abs_response('{}') + stub_vmpooler_service_response(vmpooler_only_json) allow(Open3).to receive(:capture3) - .with('floaty list --active --json') - .and_return([mock_vmpooler_json, '', instance_double(Process::Status, success?: true)]) + .with('nmap', '-Pn', '-p', '22', 'tight-mantrap.delivery.puppetlabs.net') + .and_return([vmpooler_only_nmap, '', success_status]) + end + + it 'discovers VMs not tracked by ABS' do + inventory = described_class.new + result = inventory.generate + + expect(result['targets'].length).to eq(1) + expect(result['targets'].first['name']).to eq('tight-mantrap') + expect(result['targets'].first['vars']['type']).to eq('ubuntu-2404-x86_64') + end + end + + describe 'host filtering with nmap' do + before do + stub_abs_response(mock_abs_json) + stub_vmpooler_service_response(mock_vmpooler_service_json) end context 'when some hosts are unreachable' do before do allow(Open3).to receive(:capture3) .with('nmap', '-Pn', '-p', '22', *hostnames) - .and_return([mock_nmap_output_partial, mock_nmap_stderr, instance_double(Process::Status, success?: true)]) + .and_return([mock_nmap_output_partial, mock_nmap_stderr, success_status]) end - it 'filters out unreachable hosts' do + it 'moves unreachable hosts to the unavailable group' do inventory = described_class.new result = inventory.generate - expect(result['targets'].length).to eq(2) - expect(result['targets'].map { |t| t['name'] }).to contain_exactly('tender-punditry', 'normal-meddling') + expect(result['targets'].length).to eq(3) - # Windows group should be empty since onetime-algebra is unreachable windows_group = result['groups'].find { |g| g['name'] == 'windows' } expect(windows_group['targets']).to eq([]) - # Linux group should only contain reachable hosts linux_group = result['groups'].find { |g| g['name'] == 'linux' } expect(linux_group['targets']).to contain_exactly('tender-punditry', 'normal-meddling') + + unavailable_group = result['groups'].find { |g| g['name'] == 'unavailable' } + expect(unavailable_group['targets']).to contain_exactly('onetime-algebra') + expect(unavailable_group['facts']).to eq('role' => 'unavailable') end end @@ -366,16 +549,21 @@ before do allow(Open3).to receive(:capture3) .with('nmap', '-Pn', '-p', '22', *hostnames) - .and_return(['', mock_nmap_stderr, instance_double(Process::Status, success?: true)]) + .and_return(['', mock_nmap_stderr, success_status]) end - it 'generates inventory with empty targets' do + it 'puts all targets in the unavailable group' do inventory = described_class.new result = inventory.generate - expect(result['targets']).to eq([]) + expect(result['targets'].length).to eq(3) expect(result['groups'].find { |g| g['name'] == 'windows' }['targets']).to eq([]) expect(result['groups'].find { |g| g['name'] == 'linux' }['targets']).to eq([]) + + unavailable_group = result['groups'].find { |g| g['name'] == 'unavailable' } + expect(unavailable_group['targets']).to contain_exactly( + 'onetime-algebra', 'tender-punditry', 'normal-meddling' + ) end end end