diff --git a/openc3-cosmos-cmd-tlm-api/app/models/streaming_api.rb b/openc3-cosmos-cmd-tlm-api/app/models/streaming_api.rb index bd2fe62bbd..3fe523d946 100644 --- a/openc3-cosmos-cmd-tlm-api/app/models/streaming_api.rb +++ b/openc3-cosmos-cmd-tlm-api/app/models/streaming_api.rb @@ -47,13 +47,23 @@ def build_item_collection(collection, data, start_time: nil, end_time: nil, scop # Unwrap LATEST packet into individual items parsed = StreamingKey.parse(key, item_key: true) if parsed.packet_name == 'LATEST' + # The item to packet map only covers telemetry + raise "LATEST is only supported for telemetry: #{key}" if parsed.cmd_or_tlm == :CMD item_map = OpenC3::TargetModel.get_item_to_packet_map(parsed.target_name, scope: scope) packet_names = item_map[parsed.item_name] raise "Item '#{parsed.target_name} LATEST #{parsed.item_name}' does not exist for scope: #{scope}" unless packet_names + # Skip packets the user can't access unless they can't access any of them + auth_error = nil + added = false packet_names.each do |packet_name| new_key = parsed.with(packet_name: packet_name).to_key_string - collection.add(StreamingObject.new(new_key, start_time, end_time, item_key: item_key, scope: scope, token: token)) + collection.add(StreamingObject.new(new_key, start_time, end_time, item_key: item_key, source_key: key, scope: scope, token: token)) + added = true + rescue OpenC3::AuthError, OpenC3::ForbiddenError => e + OpenC3::Logger.info("Skipping unauthorized LATEST packet: #{new_key}") + auth_error = e end + raise auth_error if auth_error and not added else collection.add(StreamingObject.new(key, start_time, end_time, item_key: item_key, scope: scope, token: token)) end diff --git a/openc3-cosmos-cmd-tlm-api/app/models/streaming_object.rb b/openc3-cosmos-cmd-tlm-api/app/models/streaming_object.rb index 7ecffe61d8..5592e6dbb1 100644 --- a/openc3-cosmos-cmd-tlm-api/app/models/streaming_object.rb +++ b/openc3-cosmos-cmd-tlm-api/app/models/streaming_object.rb @@ -40,11 +40,14 @@ class StreamingObject attr_reader :id attr_reader :realtime attr_reader :item_key + # The key as subscribed, which differs from key when a LATEST item expands to a concrete packet + attr_reader :source_key - def initialize(key, start_time_nsec, end_time_nsec, item_key: nil, scope:, token: nil) + def initialize(key, start_time_nsec, end_time_nsec, item_key: nil, source_key: nil, scope:, token: nil) key = key.upcase @key = key @item_key = item_key + @source_key = source_key ? source_key.upcase : key parsed = StreamingKey.parse(key, item_key: !!item_key) @stream_mode = parsed.stream_mode @cmd_or_tlm = parsed.cmd_or_tlm @@ -76,7 +79,9 @@ def initialize(key, start_time_nsec, end_time_nsec, item_key: nil, scope:, token @offset = "0-0" @offset = OpenC3::Topic.get_last_offset(@topic, db_shard: @db_shard) unless @start_time if @item_key - @id = 'ITEM__' + key + # Include the source_key and item_key so a LATEST item (which expands to concrete + # packet keys) doesn't collide with an explicit subscription to the same packet item + @id = "ITEM__#{key}__#{@source_key}__#{@item_key}" else @id = 'PACKET__' + key end diff --git a/openc3-cosmos-cmd-tlm-api/app/models/streaming_object_collection.rb b/openc3-cosmos-cmd-tlm-api/app/models/streaming_object_collection.rb index c2c7b5e5ca..2398241bd2 100644 --- a/openc3-cosmos-cmd-tlm-api/app/models/streaming_object_collection.rb +++ b/openc3-cosmos-cmd-tlm-api/app/models/streaming_object_collection.rb @@ -58,36 +58,54 @@ def add(object) def remove(object) @mutex.synchronize do found_object = @objects_by_id[object.id] - if found_object - @objects.delete(found_object) - @objects_by_id.delete(found_object.id) - item_objects = @item_objects_by_topic[object.topic] - item_objects.delete(found_object) if item_objects - packet_objects = @packet_objects_by_topic[object.topic] - packet_objects.delete(found_object) if packet_objects - if item_objects - if item_objects.length == 0 - if packet_objects - if packet_objects.length == 0 - # Nothing left in either for this topic - @topics_and_offsets.delete(object.topic) - end - else - # Just item_objects and nothing left - @topics_and_offsets.delete(object.topic) - end - end - else - if packet_objects - if packet_objects.length == 0 - # Just packet_objects and nothing left - @topics_and_offsets.delete(object.topic) - end - else - # Neither objects - Shouldn't happen + # Items can be added as [key, item_key] pairs but removed by key alone + # (the item_key defaults to the key) so remove every item object subscribed with the same key. + # Match on source_key so removing an explicit item doesn't also remove a + # LATEST subscription which expanded to the same concrete key (and vice versa) + key_only = (object.item_key and object.item_key.upcase == object.source_key) + if object.item_key and (key_only or not found_object) + # An explicit item_key that wasn't found exactly must also match on item_key + # so an unknown [key, item_key] pair can't remove other subscriptions + @objects.select do |obj| + obj.item_key and obj.source_key == object.source_key and (key_only or obj.item_key == object.item_key) + end.each do |obj| + _remove_found(obj) + end + elsif found_object + _remove_found(found_object) + end + end + end + + # Must be called with @mutex held + def _remove_found(object) + @objects.delete(object) + @objects_by_id.delete(object.id) + item_objects = @item_objects_by_topic[object.topic] + item_objects.delete(object) if item_objects + packet_objects = @packet_objects_by_topic[object.topic] + packet_objects.delete(object) if packet_objects + if item_objects + if item_objects.length == 0 + if packet_objects + if packet_objects.length == 0 + # Nothing left in either for this topic @topics_and_offsets.delete(object.topic) end + else + # Just item_objects and nothing left + @topics_and_offsets.delete(object.topic) + end + end + else + if packet_objects + if packet_objects.length == 0 + # Just packet_objects and nothing left + @topics_and_offsets.delete(object.topic) end + else + # Neither objects - Shouldn't happen + @topics_and_offsets.delete(object.topic) end end end diff --git a/openc3-cosmos-cmd-tlm-api/spec/models/streaming_api_spec.rb b/openc3-cosmos-cmd-tlm-api/spec/models/streaming_api_spec.rb index b5ecc03e57..2b1773de7c 100644 --- a/openc3-cosmos-cmd-tlm-api/spec/models/streaming_api_spec.rb +++ b/openc3-cosmos-cmd-tlm-api/spec/models/streaming_api_spec.rb @@ -555,6 +555,124 @@ def start_fake_thread(stop_delay) end end + describe '#build_item_collection' do + before(:each) do + allow(OpenC3::TargetModel).to receive(:get_item_to_packet_map).with('INST', scope: 'DEFAULT').and_return({ + 'TEMP1' => ['HEALTH_STATUS', 'PARAMS'], + }) + end + + let(:latest_key) { 'DECOM__TLM__INST__LATEST__TEMP1__CONVERTED' } + let(:explicit_key) { 'DECOM__TLM__INST__HEALTH_STATUS__TEMP1__CONVERTED' } + + def build(items) + collection = StreamingObjectCollection.new + @api.build_item_collection(collection, { 'items' => items }, scope: 'DEFAULT') + collection + end + + it 'expands LATEST into one object per packet keyed by the LATEST key' do + collection = build([latest_key]) + expect(collection.objects.map { |o| [o.key, o.item_key] }).to eq([ + ['DECOM__TLM__INST__HEALTH_STATUS__TEMP1__CONVERTED', latest_key], + ['DECOM__TLM__INST__PARAMS__TEMP1__CONVERTED', latest_key], + ]) + end + + it 'keeps a LATEST item and an explicit item for the same packet' do + collection = build([latest_key, explicit_key]) + expect(collection.length).to eq(3) + expect(collection.objects.select { |o| o.key == explicit_key }.map(&:item_key)).to contain_exactly(latest_key, explicit_key) + end + + it 'removing the LATEST item leaves the explicit item' do + collection = build([latest_key, explicit_key]) + build([latest_key]).objects.each { |o| collection.remove(o) } + expect(collection.objects.map { |o| [o.key, o.item_key] }).to eq([[explicit_key, explicit_key]]) + end + + it 'removing the explicit item leaves the LATEST item' do + collection = build([latest_key, explicit_key]) + build([explicit_key]).objects.each { |o| collection.remove(o) } + expect(collection.objects.map(&:item_key).uniq).to eq([latest_key]) + expect(collection.length).to eq(2) + end + + it 'rejects LATEST for commands' do + expect { build(['DECOM__CMD__INST__LATEST__TEMP1__CONVERTED']) }.to raise_error(/LATEST is only supported for telemetry/) + end + + it 'skips LATEST packets the user cannot access' do + allow_any_instance_of(StreamingObject).to receive(:authorize) do |_object, packet_name:, **| + raise OpenC3::ForbiddenError, 'forbidden' if packet_name == 'PARAMS' + end + expect(build([latest_key]).objects.map(&:packet_name)).to eq(['HEALTH_STATUS']) + end + + it 'raises when the user cannot access any LATEST packet' do + allow_any_instance_of(StreamingObject).to receive(:authorize).and_raise(OpenC3::ForbiddenError, 'forbidden') + expect { build([latest_key]) }.to raise_error(OpenC3::ForbiddenError) + end + + it 'removes an item added with an item_key when removed by key alone' do + collection = build([[explicit_key, '0']]) + build([explicit_key]).objects.each { |o| collection.remove(o) } + expect(collection).to be_empty + end + + it 'removing an explicit item by key alone leaves the LATEST item' do + collection = build([latest_key, [explicit_key, '0']]) + build([explicit_key]).objects.each { |o| collection.remove(o) } + expect(collection.objects.map(&:item_key).uniq).to eq([latest_key]) + expect(collection.length).to eq(2) + end + + it 'removing a LATEST item by key alone leaves the explicit item' do + collection = build([[latest_key, '0'], explicit_key]) + build([latest_key]).objects.each { |o| collection.remove(o) } + expect(collection.objects.map { |o| [o.key, o.item_key] }).to eq([[explicit_key, explicit_key]]) + end + + it 'removes every item_key for a key when removed by key alone' do + collection = build([explicit_key, [explicit_key, '0']]) + expect(collection.length).to eq(2) + build([explicit_key]).objects.each { |o| collection.remove(o) } + expect(collection).to be_empty + end + + it 'removing a key with an item_key only removes that item_key' do + collection = build([explicit_key, [explicit_key, '0']]) + build([[explicit_key, '0']]).objects.each { |o| collection.remove(o) } + expect(collection.objects.map(&:item_key)).to eq([explicit_key]) + end + + it 'removing an unknown item_key does not remove other item_keys' do + collection = build([explicit_key, [explicit_key, '0']]) + build([[explicit_key, '1']]).objects.each { |o| collection.remove(o) } + expect(collection.objects.map(&:item_key)).to contain_exactly(explicit_key, '0') + end + + it 'falls back to matching source_key and item_key when the exact object is not found' do + allow(OpenC3::TargetModel).to receive(:get_item_to_packet_map).with('INST', scope: 'DEFAULT').and_return({ + 'TEMP1' => ['PARAMS'], + }) + collection = build([[latest_key, '0'], [latest_key, '1']]) + allow(OpenC3::TargetModel).to receive(:get_item_to_packet_map).with('INST', scope: 'DEFAULT').and_return({ + 'TEMP1' => ['HEALTH_STATUS'], + }) + # The HEALTH_STATUS object was never added so the removal falls back to the sweep + build([[latest_key, '0']]).objects.each { |o| collection.remove(o) } + expect(collection.objects.map(&:item_key)).to eq(['1']) + end + + it 'keeps a LATEST item and an explicit item with the same item_key' do + collection = build([[latest_key, '0'], [explicit_key, '0']]) + expect(collection.length).to eq(3) + build([[latest_key, '0']]).objects.each { |o| collection.remove(o) } + expect(collection.objects.map { |o| [o.key, o.source_key] }).to eq([[explicit_key, explicit_key]]) + end + end + describe 'add with COSMOS_ALL packets and authorization' do it 'skips unauthorized packets when using COSMOS_ALL' do $openc3_authorize = true diff --git a/openc3/lib/openc3/api/limits_api.rb b/openc3/lib/openc3/api/limits_api.rb index d233536de0..f7d4e15c51 100644 --- a/openc3/lib/openc3/api/limits_api.rb +++ b/openc3/lib/openc3/api/limits_api.rb @@ -121,29 +121,16 @@ def limits_enabled?(*args, manual: false, scope: $openc3_scope, token: $openc3_t # # Favor the first syntax where possible as it is more succinct. # - # @param args [String|Array] See the description for calling style + # @param args [String|Array] See the description for calling style. + # A packet name of LATEST applies to every packet which contains the item. def enable_limits(*args, manual: false, scope: $openc3_scope, token: $openc3_token) - target_name, packet_name, item_name = _tlm_process_args(args, 'enable_limits', scope: scope) - authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) - packet = TargetModel.packet(target_name, packet_name, scope: scope) - found_item = nil - packet['items'].each do |item| - if item['name'] == item_name - item['limits']['enabled'] = true - found_item = item - break - end + target_name, packet_names, item_name = _tlm_process_args_all_packets(args, 'enable_limits', scope: scope) + packet_names.each do |packet_name| + authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) + end + packet_names.each do |packet_name| + _set_limits_enabled(target_name, packet_name, item_name, true, scope: scope) end - raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist while enabling limits" unless found_item - - TargetModel.set_packet(target_name, packet_name, packet, scope: scope) - - message = "Enabling Limits For '#{target_name} #{packet_name} #{item_name}'" - Logger.info(message, scope: scope) - - event = { type: :LIMITS_ENABLE_STATE, target_name: target_name, packet_name: packet_name, - item_name: item_name, enabled: true, time_nsec: Time.now.to_nsec_from_epoch, message: message } - LimitsEventTopic.write(event, scope: scope) end # Disable limit checking for a telemetry item @@ -154,29 +141,16 @@ def enable_limits(*args, manual: false, scope: $openc3_scope, token: $openc3_tok # # Favor the first syntax where possible as it is more succinct. # - # @param args [String|Array] See the description for calling style + # @param args [String|Array] See the description for calling style. + # A packet name of LATEST applies to every packet which contains the item. def disable_limits(*args, manual: false, scope: $openc3_scope, token: $openc3_token) - target_name, packet_name, item_name = _tlm_process_args(args, 'disable_limits', scope: scope) - authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) - packet = TargetModel.packet(target_name, packet_name, scope: scope) - found_item = nil - packet['items'].each do |item| - if item['name'] == item_name - item['limits']['enabled'] = false - found_item = item - break - end + target_name, packet_names, item_name = _tlm_process_args_all_packets(args, 'disable_limits', scope: scope) + packet_names.each do |packet_name| + authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) + end + packet_names.each do |packet_name| + _set_limits_enabled(target_name, packet_name, item_name, false, scope: scope) end - raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist while disabling limits" unless found_item - - TargetModel.set_packet(target_name, packet_name, packet, scope: scope) - - message = "Disabling Limits for '#{target_name} #{packet_name} #{item_name}'" - Logger.info(message, scope: scope) - - event = { type: :LIMITS_ENABLE_STATE, target_name: target_name, packet_name: packet_name, - item_name: item_name, enabled: false, time_nsec: Time.now.to_nsec_from_epoch, message: message } - LimitsEventTopic.write(event, scope: scope) end # Get a Hash of all the limits sets defined for an item. Hash keys are the limit @@ -189,9 +163,16 @@ def disable_limits(*args, manual: false, scope: $openc3_scope, token: $openc3_to # # @return [Hash{String => Array}] def get_limits(target_name, packet_name, item_name, cache_timeout: nil, manual: false, scope: $openc3_scope, token: $openc3_token) + target_name = target_name.upcase + packet_name = packet_name.upcase + item_name = item_name.upcase + # Resolve LATEST before authorizing so the real packet is authorized + if packet_name == 'LATEST' + packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: cache_timeout, scope: scope) + end authorize(permission: 'tlm', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) limits = {} - item = _get_item(target_name, packet_name, item_name, cache_timeout: cache_timeout, scope: scope) + item = TargetModel.packet_item(target_name, packet_name, item_name, scope: scope) item['limits'].each do |key, vals| next unless vals.is_a?(Hash) @@ -202,56 +183,58 @@ def get_limits(target_name, packet_name, item_name, cache_timeout: nil, manual: end # Change the limits settings for a given item. By default, a new limits set called 'CUSTOM' - # is created to avoid overriding existing limits. + # is created to avoid overriding existing limits. A packet name of LATEST + # applies to every packet which contains the item. def set_limits(target_name, packet_name, item_name, red_low, yellow_low, yellow_high, red_high, green_low = nil, green_high = nil, limits_set = 'CUSTOM', persistence = nil, enabled = true, manual: false, scope: $openc3_scope, token: $openc3_token) - authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) + target_name = target_name.upcase + packet_name = packet_name.upcase + item_name = item_name.upcase + packet_names = (packet_name == 'LATEST') ? _tlm_packet_names(target_name, packet_name, item_name, scope: scope) : [packet_name] + packet_names.each do |name| + authorize(permission: 'tlm_set', target_name: target_name, packet_name: name, manual: manual, scope: scope, token: token) + end if (red_low > yellow_low) || (yellow_low >= yellow_high) || (yellow_high > red_high) raise "Invalid limits specified. Ensure yellow limits are within red limits." end if (green_low && green_high) && ((yellow_low > green_low) || (green_low >= green_high) || (green_high > yellow_high)) raise "Invalid limits specified. Ensure green limits are within yellow limits." end - packet = TargetModel.packet(target_name, packet_name, scope: scope) - found_item = nil - packet['items'].each do |item| - if item['name'] == item_name - if item['limits'] - item['limits']['persistence_setting'] = persistence if persistence - if enabled - item['limits']['enabled'] = true - else - item['limits']['enabled'] = false - end - limits = {} - limits['red_low'] = red_low - limits['yellow_low'] = yellow_low - limits['yellow_high'] = yellow_high - limits['red_high'] = red_high - limits['green_low'] = green_low if green_low && green_high - limits['green_high'] = green_high if green_low && green_high - item['limits'][limits_set] = limits - found_item = item - break - else - raise "Cannot set_limits on item without any limits" - end - end + # Validate every packet before changing any so an error doesn't leave a partial update. + # TargetModel.packet returns a cached hash so it must not be modified until validated. + items = packet_names.map do |name| + packet = TargetModel.packet(target_name, name, scope: scope) + item = packet['items'].find { |packet_item| packet_item['name'] == item_name } + raise "Item '#{target_name} #{name} #{item_name}' does not exist while setting limits" unless item + raise "Cannot set_limits on item without any limits" unless item['limits'] + [name, packet, item] + end + items.each do |name, packet, item| + item['limits']['persistence_setting'] = persistence if persistence + item['limits']['enabled'] = enabled ? true : false + limits = {} + limits['red_low'] = red_low + limits['yellow_low'] = yellow_low + limits['yellow_high'] = yellow_high + limits['red_high'] = red_high + limits['green_low'] = green_low if green_low && green_high + limits['green_high'] = green_high if green_low && green_high + item['limits'][limits_set] = limits + + message = "Setting '#{target_name} #{name} #{item_name}' limits to #{red_low} #{yellow_low} #{yellow_high} #{red_high}" + message << " #{green_low} #{green_high}" if green_low && green_high + message << " in set #{limits_set} with persistence #{persistence} as enabled #{enabled}" + Logger.info(message, scope: scope) + + TargetModel.set_packet(target_name, name, packet, scope: scope) + + event = { type: :LIMITS_SETTINGS, target_name: target_name, packet_name: name, + item_name: item_name, red_low: red_low, yellow_low: yellow_low, yellow_high: yellow_high, red_high: red_high, + green_low: green_low, green_high: green_high, limits_set: limits_set, persistence: persistence, enabled: enabled, + time_nsec: Time.now.to_nsec_from_epoch, message: message } + LimitsEventTopic.write(event, scope: scope) end - raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist while setting limits" unless found_item - message = "Setting '#{target_name} #{packet_name} #{item_name}' limits to #{red_low} #{yellow_low} #{yellow_high} #{red_high}" - message << " #{green_low} #{green_high}" if green_low && green_high - message << " in set #{limits_set} with persistence #{persistence} as enabled #{enabled}" - Logger.info(message, scope: scope) - - TargetModel.set_packet(target_name, packet_name, packet, scope: scope) - - event = { type: :LIMITS_SETTINGS, target_name: target_name, packet_name: packet_name, - item_name: item_name, red_low: red_low, yellow_low: yellow_low, yellow_high: yellow_high, red_high: red_high, - green_low: green_low, green_high: green_high, limits_set: limits_set, persistence: persistence, enabled: enabled, - time_nsec: Time.now.to_nsec_from_epoch, message: message } - LimitsEventTopic.write(event, scope: scope) end # Change the color associated with a telemetry item state in realtime. @@ -266,9 +249,16 @@ def set_limits(target_name, packet_name, item_name, red_low, yellow_low, yellow_ # or RED. Pass nil to clear (remove) the state color. Setting a color also # enables limits for the item, but clearing the color (nil) does not change # whether limits are enabled. + # A packet name of LATEST applies to every packet which contains the item. def set_state_color(target_name, packet_name, item_name, state_name, color, manual: false, scope: $openc3_scope, token: $openc3_token) - authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) + target_name = target_name.upcase + packet_name = packet_name.upcase + item_name = item_name.upcase + packet_names = (packet_name == 'LATEST') ? _tlm_packet_names(target_name, packet_name, item_name, scope: scope) : [packet_name] + packet_names.each do |name| + authorize(permission: 'tlm_set', target_name: target_name, packet_name: name, manual: manual, scope: scope, token: token) + end state_name = state_name.to_s.upcase unless color.nil? color = color.to_s.upcase @@ -276,40 +266,42 @@ def set_state_color(target_name, packet_name, item_name, state_name, color, raise "Invalid state color '#{color}'. Must be one of #{PacketItem::STATE_COLORS.join(', ')}." end end - packet = TargetModel.packet(target_name, packet_name, scope: scope) - found_item = nil - packet['items'].each do |item| - if item['name'] == item_name - unless item['states'] && item['states'][state_name] - raise "State '#{state_name}' does not exist for item '#{target_name} #{packet_name} #{item_name}'" - end - if color.nil? - # Clear the state color - item['states'][state_name].delete('color') - else - item['states'][state_name]['color'] = color - # Defining a state color implies limits are enabled for the item - item['limits'] ||= {} - item['limits']['enabled'] = true - end - found_item = item - break + # Validate every packet before changing any so an error doesn't leave a partial update. + # TargetModel.packet returns a cached hash so it must not be modified until validated. + items = packet_names.map do |name| + packet = TargetModel.packet(target_name, name, scope: scope) + item = packet['items'].find { |packet_item| packet_item['name'] == item_name } + raise "Item '#{target_name} #{name} #{item_name}' does not exist while setting state color" unless item + unless item['states'] && item['states'][state_name] + raise "State '#{state_name}' does not exist for item '#{target_name} #{name} #{item_name}'" end + [name, packet, item] end - raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist while setting state color" unless found_item - if color.nil? - message = "Clearing '#{target_name} #{packet_name} #{item_name}' state #{state_name} color" - else - message = "Setting '#{target_name} #{packet_name} #{item_name}' state #{state_name} color to #{color}" - end - Logger.info(message, scope: scope) + items.each do |name, packet, item| + if color.nil? + # Clear the state color + item['states'][state_name].delete('color') + else + item['states'][state_name]['color'] = color + # Defining a state color implies limits are enabled for the item + item['limits'] ||= {} + item['limits']['enabled'] = true + end - TargetModel.set_packet(target_name, packet_name, packet, scope: scope) + if color.nil? + message = "Clearing '#{target_name} #{name} #{item_name}' state #{state_name} color" + else + message = "Setting '#{target_name} #{name} #{item_name}' state #{state_name} color to #{color}" + end + Logger.info(message, scope: scope) - event = { type: :LIMITS_STATE_COLOR, target_name: target_name, packet_name: packet_name, - item_name: item_name, state_name: state_name, color: color, - time_nsec: Time.now.to_nsec_from_epoch, message: message } - LimitsEventTopic.write(event, scope: scope) + TargetModel.set_packet(target_name, name, packet, scope: scope) + + event = { type: :LIMITS_STATE_COLOR, target_name: target_name, packet_name: name, + item_name: item_name, state_name: state_name, color: color, + time_nsec: Time.now.to_nsec_from_epoch, message: message } + LimitsEventTopic.write(event, scope: scope) + end end # Returns all limits_groups and their members @@ -448,17 +440,25 @@ def _limits_group(group_name, action:, manual:, scope:, token:) end end - # Gets an item. The code below is mostly duplicated from tlm_process_args in tlm_api.rb. - # - # @param target_name [String] target name - # @param packet_name [String] packet name - # @param item_name [String] item name - # @param scope [String] scope - # @return Hash The requested item based on the packet name - def _get_item(target_name, packet_name, item_name, cache_timeout: nil, scope:) - # Determine if this item exists, it will raise appropriate errors if not - packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: cache_timeout, scope: $openc3_scope) if packet_name == 'LATEST' - return TargetModel.packet_item(target_name, packet_name, item_name, scope: scope) + def _set_limits_enabled(target_name, packet_name, item_name, enabled, scope:) + packet = TargetModel.packet(target_name, packet_name, scope: scope) + item = packet['items'].find { |packet_item| packet_item['name'] == item_name } + action = enabled ? 'enabling' : 'disabling' + raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist while #{action} limits" unless item + + item['limits']['enabled'] = enabled + TargetModel.set_packet(target_name, packet_name, packet, scope: scope) + + if enabled + message = "Enabling Limits For '#{target_name} #{packet_name} #{item_name}'" + else + message = "Disabling Limits for '#{target_name} #{packet_name} #{item_name}'" + end + Logger.info(message, scope: scope) + + event = { type: :LIMITS_ENABLE_STATE, target_name: target_name, packet_name: packet_name, + item_name: item_name, enabled: enabled, time_nsec: Time.now.to_nsec_from_epoch, message: message } + LimitsEventTopic.write(event, scope: scope) end end end diff --git a/openc3/lib/openc3/api/tlm_api.rb b/openc3/lib/openc3/api/tlm_api.rb index de97b4fc79..2711853f16 100644 --- a/openc3/lib/openc3/api/tlm_api.rb +++ b/openc3/lib/openc3/api/tlm_api.rb @@ -119,6 +119,7 @@ def set_tlm(*args, type: :CONVERTED, manual: false, cache_timeout: nil, scope: $ # @param type [Symbol] Telemetry type, :RAW, :CONVERTED (default), :FORMATTED # @param received_time [Integer, nil] Optional received time as nanoseconds since Unix epoch def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, received_time: nil, manual: false, scope: $openc3_scope, token: $openc3_token) + _tlm_reject_latest(packet_name.to_s.upcase, 'inject_tlm') authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) type = type.to_s.intern target_name = target_name.upcase @@ -174,12 +175,17 @@ def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, stor # # @param args The args must either be a string followed by a value or # three strings followed by a value (see the calling style in the - # description). + # description). A packet name of LATEST overrides the item in every + # packet which contains it. # @param type [Symbol] Telemetry type, :ALL (default), :RAW, :CONVERTED, :FORMATTED def override_tlm(*args, type: :ALL, manual: false, scope: $openc3_scope, token: $openc3_token) - target_name, packet_name, item_name, value = _set_tlm_process_args(args, __method__, scope: scope) - authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) - CvtModel.override(target_name, packet_name, item_name, value, type: type.intern, scope: scope) + target_name, packet_names, item_name, value = _set_tlm_process_args_all_packets(args, __method__, scope: scope) + packet_names.each do |packet_name| + authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) + end + packet_names.each do |packet_name| + CvtModel.override(target_name, packet_name, item_name, value, type: type.intern, scope: scope) + end end # Get the list of CVT overrides @@ -198,13 +204,18 @@ def get_overrides(manual: false, scope: $openc3_scope, token: $openc3_token) # Favor the first syntax where possible as it is more succinct. # # @param args The args must either be a string or three strings - # (see the calling style in the description). + # (see the calling style in the description). A packet name of LATEST + # normalizes the item in every packet which contains it. # @param type [Symbol] Telemetry type, :ALL (default), :RAW, :CONVERTED, :FORMATTED # Also takes :ALL which means to normalize all telemetry types def normalize_tlm(*args, type: :ALL, manual: false, scope: $openc3_scope, token: $openc3_token) - target_name, packet_name, item_name = _tlm_process_args(args, __method__, scope: scope) - authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) - CvtModel.normalize(target_name, packet_name, item_name, type: type.intern, scope: scope) + target_name, packet_names, item_name = _tlm_process_args_all_packets(args, __method__, scope: scope) + packet_names.each do |packet_name| + authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) + end + packet_names.each do |packet_name| + CvtModel.normalize(target_name, packet_name, item_name, type: type.intern, scope: scope) + end end # Returns the raw buffer for a telemetry packet. @@ -265,45 +276,25 @@ def get_tlm_available(items, manual: false, scope: $openc3_scope, token: $openc3 packet_name = orig_packet_name raise ArgumentError, "items must be formatted as TGT__PKT__ITEM__TYPE" if target_name.nil? || packet_name.nil? || item_name.nil? || value_type.nil? if orig_packet_name == 'LATEST' - # TODO: Do we need to lookup ALL the possible packets for this item? + # A LATEST item can come from any packet containing it. Report the best value + # type and limits of any of them since the CVT falls back to the available + # type and historical lookups check the type available in each packet. # We can have a large cache_timeout of 1 because all we're trying to do is lookup a packet packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: 1, scope: scope) + packet_names = _tlm_packet_names(target_name, orig_packet_name, item_name, scope: scope) + else + packet_names = [packet_name] + end + # Authorize every packet whose definition is read below, not just the latest one + (packet_names | [packet_name]).each do |name| + authorize(permission: 'tlm', target_name: target_name, packet_name: name, scope: scope, token: token) end - authorize(permission: 'tlm', target_name: target_name, packet_name: packet_name, scope: scope, token: token) begin - item = TargetModel.packet_item(target_name, packet_name, item_name, scope: scope) - if Packet::RESERVED_ITEM_NAMES.include?(item_name) - value_type = 'RAW' # Must request the raw value when dealing with the reserved items - end - - # Arrays must be accessed as RAW since there's no conversion - if item['array_size'] - value_type = 'RAW' - end - - case value_type - when 'FORMATTED', 'WITH_UNITS' - if item['format_string'] or item['units'] - results << [target_name, orig_packet_name, item_name, 'FORMATTED'].join('__') - # This logic must match the logic in Packet#decom - elsif item['states'] or (item['read_conversion'] and item['data_type'] != 'DERIVED') - results << [target_name, orig_packet_name, item_name, 'CONVERTED'].join('__') - else - results << [target_name, orig_packet_name, item_name, 'RAW'].join('__') - end - when 'CONVERTED' - # This logic must match the logic in Packet#decom - if item['states'] or (item['read_conversion'] and item['data_type'] != 'DERIVED') - results << [target_name, orig_packet_name, item_name, 'CONVERTED'].join('__') - else - results << [target_name, orig_packet_name, item_name, 'RAW'].join('__') - end - else # RAW or unknown - results << [target_name, orig_packet_name, item_name, 'RAW'].join('__') - end - + packet_items = packet_names.map { |name| TargetModel.packet_item(target_name, name, item_name, scope: scope) } + available_type = packet_items.map { |packet_item| _tlm_available_value_type(packet_item, item_name, value_type) }.max_by { |type| VALUE_TYPE_ORDER.index(type) } + results << [target_name, orig_packet_name, item_name, available_type].join('__') # Tack on __LIMITS to notify that we have an available limits value - if item['limits']['DEFAULT'] + if packet_items.any? { |packet_item| packet_item['limits']['DEFAULT'] } results[-1] += '__LIMITS' end rescue RuntimeError @@ -341,7 +332,20 @@ def get_tlm_values(items, stale_time: 30, cache_timeout: nil, manual: false, sta parts = item.to_s.upcase.split('__') raise ArgumentError, "items must be formatted as TGT__PKT__ITEM__TYPE" if parts.length < 4 || parts.length > 5 target_name, packet_name, item_name, value_type, limits = parts - if packet_name == 'LATEST' # Lookup packet_name in case of LATEST + if packet_name == 'LATEST' + if start_time + # The packet received most recently now isn't necessarily the one received most + # recently at start_time so keep every candidate packet for _tlm_latest_history + # Each packet must be queried for the value type and limits it has or the query fails + candidates = _tlm_packet_names(target_name, packet_name, item_name, scope: scope).map do |candidate| + packets << [target_name, candidate] + candidate_item = TargetModel.packet_item(target_name, candidate, item_name, scope: scope) + candidate_limits = (limits and candidate_item['limits']['DEFAULT']) ? limits : nil + [target_name, candidate, item_name, _tlm_available_value_type(candidate_item, item_name, value_type), candidate_limits] + end + cvt_items << [target_name, packet_name, item_name, value_type, limits, candidates] + next + end packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: cache_timeout, scope: scope) end # NOTE: limits is required by the historical (start_time) QuestDB lookup @@ -352,6 +356,9 @@ def get_tlm_values(items, stale_time: 30, cache_timeout: nil, manual: false, sta packets.each do |target_name, packet_name| authorize(permission: 'tlm', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) end + if cvt_items.any? { |cvt_item| cvt_item.length == 6 } + return _tlm_latest_history(cvt_items, start_time: start_time, end_time: end_time, cache_timeout: cache_timeout, scope: scope) + end CvtModel.get_tlm_values(cvt_items, stale_time: stale_time, cache_timeout: cache_timeout, start_time: start_time, end_time: end_time, scope: scope) end @@ -446,6 +453,7 @@ def subscribe_packets(packets, manual: false, scope: $openc3_scope, token: $open packets.each do |target_name, packet_name| target_name = target_name.upcase packet_name = packet_name.upcase + _tlm_reject_latest(packet_name, 'subscribe_packets') authorize(permission: 'tlm', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token) topic = "#{scope}__DECOM__{#{target_name}}__#{packet_name}" db_shard = Store.db_shard_for_target(target_name, scope: scope) @@ -551,9 +559,15 @@ def _extract_target_packet_names(method_name, *args) if target_name.nil? or packet_name.nil? raise "ERROR: Both target name and packet name required. Usage: #{method_name}(\"TGT PKT\") or #{method_name}(\"TGT\", \"PKT\")" end + _tlm_reject_latest(packet_name, method_name) return [target_name, packet_name] end + # LATEST picks a packet per item so it can't identify a whole packet + def _tlm_reject_latest(packet_name, method_name) + raise "ERROR: LATEST is not supported by #{method_name}(), specify a packet name" if packet_name == 'LATEST' + end + def _extract_target_packet_item_names(method_name, *args) target_name = nil packet_name = nil @@ -589,6 +603,31 @@ def _validate_tlm_type(type) end def _tlm_process_args(args, method_name, cache_timeout: nil, scope: $openc3_scope, token: $openc3_token) + target_name, packet_name, item_name = _tlm_extract_args(args, method_name) + packet_name = _tlm_resolve_packet_name(target_name, packet_name, item_name, cache_timeout: cache_timeout, scope: scope) + return [target_name, packet_name, item_name] + end + + def _set_tlm_process_args(args, method_name, cache_timeout: nil, scope: $openc3_scope, token: $openc3_token) + target_name, packet_name, item_name, value = _set_tlm_extract_args(args, method_name) + packet_name = _tlm_resolve_packet_name(target_name, packet_name, item_name, cache_timeout: cache_timeout, scope: scope) + return [target_name, packet_name, item_name, value] + end + + # Like _tlm_process_args but returns an array of packet names where LATEST + # expands to every packet containing the item. Used by APIs which change + # an item so the change applies wherever the item appears. + def _tlm_process_args_all_packets(args, method_name, scope: $openc3_scope) + target_name, packet_name, item_name = _tlm_extract_args(args, method_name) + return [target_name, _tlm_packet_names(target_name, packet_name, item_name, scope: scope), item_name] + end + + def _set_tlm_process_args_all_packets(args, method_name, scope: $openc3_scope) + target_name, packet_name, item_name, value = _set_tlm_extract_args(args, method_name) + return [target_name, _tlm_packet_names(target_name, packet_name, item_name, scope: scope), item_name, value] + end + + def _tlm_extract_args(args, method_name) case args.length when 1 target_name, packet_name, item_name = extract_fields_from_tlm_text(args[0]) @@ -600,21 +639,10 @@ def _tlm_process_args(args, method_name, cache_timeout: nil, scope: $openc3_scop # Invalid number of arguments raise "ERROR: Invalid number of arguments (#{args.length}) passed to #{method_name}()" end - target_name = target_name.upcase - packet_name = packet_name.upcase - item_name = item_name.upcase - - if packet_name == 'LATEST' - packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: cache_timeout, scope: scope) - else - # Determine if this item exists, it will raise appropriate errors if not - TargetModel.packet_item(target_name, packet_name, item_name, scope: scope) - end - - return [target_name, packet_name, item_name] + return [target_name.upcase, packet_name.upcase, item_name.upcase] end - def _set_tlm_process_args(args, method_name, cache_timeout: nil, scope: $openc3_scope, token: $openc3_token) + def _set_tlm_extract_args(args, method_name) case args.length when 1 target_name, packet_name, item_name, value = extract_fields_from_set_tlm_text(args[0]) @@ -627,18 +655,148 @@ def _set_tlm_process_args(args, method_name, cache_timeout: nil, scope: $openc3_ # Invalid number of arguments raise "ERROR: Invalid number of arguments (#{args.length}) passed to #{method_name}()" end - target_name = target_name.upcase - packet_name = packet_name.upcase - item_name = item_name.upcase + return [target_name.upcase, packet_name.upcase, item_name.upcase, value] + end + # Resolve LATEST to the most recently received packet containing the item + def _tlm_resolve_packet_name(target_name, packet_name, item_name, cache_timeout: nil, scope: $openc3_scope) if packet_name == 'LATEST' packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: cache_timeout, scope: scope) else # Determine if this item exists, it will raise appropriate errors if not TargetModel.packet_item(target_name, packet_name, item_name, scope: scope) end + return packet_name + end - return [target_name, packet_name, item_name, value] + VALUE_TYPE_ORDER = %w(RAW CONVERTED FORMATTED).freeze + + # Returns the value type available for an item given the requested value type + def _tlm_available_value_type(item, item_name, value_type) + # Must request the raw value when dealing with the reserved items + return 'RAW' if Packet::RESERVED_ITEM_NAMES.include?(item_name) + # Arrays must be accessed as RAW since there's no conversion + return 'RAW' if item['array_size'] + + # This logic must match the logic in Packet#decom + converted = (item['states'] or (item['read_conversion'] and item['data_type'] != 'DERIVED')) + case value_type + when 'FORMATTED', 'WITH_UNITS' + return 'FORMATTED' if item['format_string'] or item['units'] + return converted ? 'CONVERTED' : 'RAW' + when 'CONVERTED' + return converted ? 'CONVERTED' : 'RAW' + else # RAW or unknown + return 'RAW' + end + end + + # Historical lookup where some items are LATEST. Those items have a 6th element + # holding the item to look up in every packet which contains the item. + def _tlm_latest_history(cvt_items, start_time:, end_time:, cache_timeout:, scope:) + latest = cvt_items.each_index.select { |index| cvt_items[index].length == 6 } + regular = cvt_items.each_index.reject { |index| cvt_items[index].length == 6 } + + # The QuestDB lookup ASOF JOINs every packet to the rows of the first, so + # a range needs a regular item first to drive the rows. Without one keep + # the realtime behavior of using the packet received most recently now. + if end_time and regular.all? { |index| cvt_items[index][2].nil? } + cvt_items = cvt_items.map do |target_name, packet_name, item_name, value_type, limits, candidates| + next [target_name, packet_name, item_name, value_type, limits] unless packet_name == 'LATEST' + packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: cache_timeout, scope: scope) + # Use the candidate so the query asks for the value type and limits that packet has + candidates.find { |candidate| candidate[1] == packet_name } || [target_name, packet_name, item_name, value_type, limits] + end + return CvtModel.tsdb_lookup(cvt_items, start_time: start_time, end_time: end_time, scope: scope) + end + + # A packet without a table fails the query so skip those candidates + packets = latest.flat_map { |index| cvt_items[index][5].map { |candidate| candidate[0..1] } }.uniq + existing = CvtModel.tsdb_packets_with_tables(packets, scope: scope) + cvt_items = cvt_items.each_with_index.map do |cvt_item, index| + next cvt_item unless latest.include?(index) + cvt_item[0..4] + [cvt_item[5].select { |candidate| existing.include?(candidate[0..1]) }] + end + + if end_time + query = [] + query_index = {} + add = lambda { |query_item| query_index[query_item] ||= (query << query_item; query.length - 1) } + positions = Array.new(cvt_items.length) + regular.each { |index| positions[index] = add.call(cvt_items[index]) } + latest.each do |index| + positions[index] = cvt_items[index][5].map do |candidate| + target_name, packet_name = candidate + [add.call(candidate), add.call([target_name, packet_name, 'PACKET_TIMESECONDS', 'RAW', nil])] + end + end + rows = CvtModel.tsdb_lookup(query, start_time: start_time, end_time: end_time, scope: scope, flatten: false) + return rows unless rows.is_a?(Array) and rows.length > 0 + rows = rows.map do |row| + positions.map do |position| + next row[position] if position.is_a?(Integer) + # Take the candidate with the newest packet time at this row + _tlm_newest(position.map { |value_index, time_index| [row[time_index][0], row[value_index]] }) || [nil, nil] + end + end + # Match tsdb_lookup which returns a flat array for a single row + return rows.length == 1 ? rows[0] : rows + end + + # A point in time returns the last row of the first packet before start_time + # with the other packets ASOF JOINed at that row. A candidate packet received + # after that row would be missed so query each candidate packet on its own. + found = false + row = Array.new(cvt_items.length) { [nil, nil] } + unless regular.all? { |index| cvt_items[index][2].nil? } + result = CvtModel.tsdb_lookup(regular.map { |index| cvt_items[index] }, start_time: start_time, scope: scope, flatten: false) + if result.is_a?(Array) and result.length > 0 + found = true + regular.each_with_index { |index, result_index| row[index] = result[0][result_index] } + end + end + # Group the LATEST items by candidate packet so each packet is queried once + by_packet = {} + latest.each do |index| + cvt_items[index][5].each do |candidate| + target_name, packet_name = candidate + (by_packet[[target_name, packet_name]] ||= {})[index] = candidate + end + end + found_values = Hash.new { |hash, key| hash[key] = [] } + by_packet.each do |(target_name, packet_name), packet_items| + query = packet_items.values.uniq + query << [target_name, packet_name, 'PACKET_TIMESECONDS', 'RAW', nil] + result = CvtModel.tsdb_lookup(query, start_time: start_time, scope: scope, flatten: false) + next unless result.is_a?(Array) and result.length > 0 + values = result[0] + packet_items.each { |index, query_item| found_values[index] << [values[-1][0], values[query.index(query_item)]] } + end + found_values.each do |index, candidates| + value = _tlm_newest(candidates) + next unless value + found = true + row[index] = value + end + return found ? row : {} + end + + # Return the value with the newest time from [time, value] pairs, or nil + def _tlm_newest(candidates) + candidates.select { |time, _| time }.max_by { |time, _| time }&.last + end + + # Expand LATEST to every packet containing the item + def _tlm_packet_names(target_name, packet_name, item_name, scope: $openc3_scope) + if packet_name == 'LATEST' + packet_names = TargetModel.get_item_to_packet_map(target_name, scope: scope)[item_name] + raise "Item '#{target_name} LATEST #{item_name}' does not exist for scope: #{scope}" unless packet_names + return packet_names + else + # Determine if this item exists, it will raise appropriate errors if not + TargetModel.packet_item(target_name, packet_name, item_name, scope: scope) + return [packet_name] + end end end end diff --git a/openc3/lib/openc3/models/cvt_model.rb b/openc3/lib/openc3/models/cvt_model.rb index 2166c3cad4..3cab6a01f0 100644 --- a/openc3/lib/openc3/models/cvt_model.rb +++ b/openc3/lib/openc3/models/cvt_model.rb @@ -134,8 +134,13 @@ def self.get_item(target_name, packet_name, item_name, type:, cache_timeout: nil end end - def self.tsdb_lookup(items, start_time:, end_time: nil, scope: $openc3_scope) - QuestDBClient.tsdb_lookup(items, start_time: start_time, end_time: end_time, scope: scope) + def self.tsdb_lookup(items, start_time:, end_time: nil, scope: $openc3_scope, flatten: true) + QuestDBClient.tsdb_lookup(items, start_time: start_time, end_time: end_time, scope: scope, flatten: flatten) + end + + # Return the [target_name, packet_name] telemetry packets which have a TSDB table + def self.tsdb_packets_with_tables(packets, scope: $openc3_scope) + QuestDBClient.tlm_packets_with_tables(packets, scope: scope) end # Return all item values and limit state from the CVT diff --git a/openc3/lib/openc3/models/target_model.rb b/openc3/lib/openc3/models/target_model.rb index d21582240d..c518965e63 100644 --- a/openc3/lib/openc3/models/target_model.rb +++ b/openc3/lib/openc3/models/target_model.rb @@ -458,10 +458,13 @@ def self.limits_groups(scope:) end def self.get_item_to_packet_map(target_name, scope:) - cache_time, item_map = @@item_map_cache[target_name] - return item_map if item_map and (Time.now - cache_time) < ITEM_MAP_CACHE_TIMEOUT - item_map_key = "#{scope}__#{target_name}__item_to_packet_map" + # Normalize before building the key so lowercase lookups share the entry + # that update_store_item_map and undeploy refresh and invalidate target_name = target_name.upcase + # The cache is keyed by item_map_key since the same target can exist in multiple scopes + item_map_key = "#{scope}__#{target_name}__item_to_packet_map" + cache_time, item_map = @@item_map_cache[item_map_key] + return item_map if item_map and (Time.now - cache_time) < ITEM_MAP_CACHE_TIMEOUT store = store_for_target(target_name, scope: scope) json_data = store.get(item_map_key) if json_data @@ -470,7 +473,7 @@ def self.get_item_to_packet_map(target_name, scope:) item_map = build_item_to_packet_map(target_name, scope: scope) store.set(item_map_key, JSON.generate(item_map, allow_nan: true)) end - @@item_map_cache[target_name] = [Time.now, item_map] + @@item_map_cache[item_map_key] = [Time.now, item_map] return item_map end @@ -917,7 +920,7 @@ def undeploy # Delete item_map item_map_key = "#{@scope}__#{@name}__item_to_packet_map" db_shard_store.del(item_map_key) - @@item_map_cache[@name] = nil + @@item_map_cache[item_map_key] = nil topic = { kind: 'deleted', type: 'target', name: @name } # The UNKNOWN target doesn't have an associated plugin @@ -1101,7 +1104,7 @@ def update_store_item_map item_map_key = "#{@scope}__#{@name}__item_to_packet_map" item_map = self.class.build_item_to_packet_map(@name, scope: @scope) Store.instance(db_shard: @db_shard).set(item_map_key, JSON.generate(item_map, allow_nan: true)) - @@item_map_cache[@name] = [Time.now, item_map] + @@item_map_cache[item_map_key] = [Time.now, item_map] end def update_store(system, clear_old: true) diff --git a/openc3/lib/openc3/script/api_shared.rb b/openc3/lib/openc3/script/api_shared.rb index 6e3f86136a..93ea24cfff 100644 --- a/openc3/lib/openc3/script/api_shared.rb +++ b/openc3/lib/openc3/script/api_shared.rb @@ -596,6 +596,8 @@ def _wait_packet(check, quiet: false, scope: $openc3_scope, token: $openc3_token) type = (check ? 'CHECK' : 'WAIT') + # LATEST would compare the RECEIVED_COUNT of whichever packet was received most recently + raise "LATEST is not supported by wait_packet, specify a packet name" if packet_name.to_s.upcase == 'LATEST' initial_count = tlm(target_name, packet_name, 'RECEIVED_COUNT', scope: scope, token: token) # If the packet has not been received the initial_count could be nil initial_count = 0 unless initial_count diff --git a/openc3/lib/openc3/utilities/questdb_client.rb b/openc3/lib/openc3/utilities/questdb_client.rb index 3323ed4450..27d504686e 100644 --- a/openc3/lib/openc3/utilities/questdb_client.rb +++ b/openc3/lib/openc3/utilities/questdb_client.rb @@ -632,6 +632,25 @@ def self.table_has_data?(table_name, start_time, end_time, db_shard: 0) false end + # Return the telemetry packets which have a table. The TSDB microservice only + # creates tables for its initial packets so a packet added later has no table + # until it's received. If the table names can't be fetched assume they all exist. + # + # @param packets [Array] [target_name, packet_name] pairs + # @return [Set] The pairs which have a table + def self.tlm_packets_with_tables(packets, scope: "DEFAULT") + existing = Set.new + packets.group_by { |target_name, _| db_shard_for_target(target_name, scope: scope) }.each do |db_shard, shard_packets| + names = table_names(db_shard: db_shard) + shard_packets.each do |target_name, packet_name| + if names.nil? or names.include?(sanitize_table_name(target_name, packet_name, scope: scope).upcase) + existing << [target_name, packet_name] + end + end + end + existing + end + # Return the names of all tables in a single query. Used to avoid querying # (and failing on) tables that were never created, e.g. commands that have # never been sent, which is the common case for targets with thousands of packets. @@ -939,12 +958,16 @@ def self.decode_reduced_row(row) # @param start_time [String, Numeric] Start timestamp for the query # @param end_time [String, Numeric, nil] End timestamp, or nil for "latest single row" # @param scope [String] Scope name + # @param flatten [Boolean] Whether a single row result is returned as a flat array # @return [Array, Hash] Array of [value, limits_state] pairs per row, or {} if no results. - # Single-row results return a flat array; multi-row results return array of arrays. - def self.tsdb_lookup(items, start_time:, end_time: nil, scope: "DEFAULT") + # Single-row results return a flat array (unless flatten is false); multi-row results return array of arrays. + def self.tsdb_lookup(items, start_time:, end_time: nil, scope: "DEFAULT", flatten: true) # Every item is a placeholder for an item which doesn't exist, so there's # nothing to query. Return a single row of nil values, one per item. - return Array.new(items.length) { [nil, nil] } if items.all? { |item| item[2].nil? } + if items.all? { |item| item[2].nil? } + row = Array.new(items.length) { [nil, nil] } + return flatten ? row : [row] + end # Group items by db_shard number while preserving their original positions db_shard_groups = {} # db_shard => { positions: [], items: [] } @@ -959,7 +982,7 @@ def self.tsdb_lookup(items, start_time:, end_time: nil, scope: "DEFAULT") # Single-db_shard fast path (most common case) if db_shard_groups.length == 1 db_shard, group = db_shard_groups.first - return tsdb_lookup_single_db_shard(group[:items], start_time: start_time, end_time: end_time, scope: scope, db_shard: db_shard) + return tsdb_lookup_single_db_shard(group[:items], start_time: start_time, end_time: end_time, scope: scope, db_shard: db_shard, flatten: flatten) end # Cross-db_shard: execute per-db_shard queries (as arrays of rows) and merge results @@ -984,7 +1007,7 @@ def self.tsdb_lookup(items, start_time:, end_time: nil, scope: "DEFAULT") end end # Match the single db_shard behavior of returning a flat array for a single row - return merged[0] if max_rows == 1 + return merged[0] if flatten and max_rows == 1 merged end @@ -1000,6 +1023,14 @@ def self.tsdb_lookup_single_db_shard(items, start_time:, end_time: nil, scope: " calculated_items = {} needed_timestamps = {} current_position = 0 + # Columns with the same name in different tables come back renamed by QuestDB + # (e.g. VALUE and VALUE_2) so alias every column and map it back when decoding + aliases = {} + add_column = lambda do |col_name| + alias_name = "__col#{aliases.length}" + aliases[alias_name] = col_name + names << "\"#{col_name}\" as #{alias_name}" + end items.each do |item| target_name, packet_name, orig_item_name, value_type, limits = item @@ -1014,7 +1045,7 @@ def self.tsdb_lookup_single_db_shard(items, start_time:, end_time: nil, scope: " index = tables.find_index {|k,_v| k == table_name } if STORED_TIMESTAMP_ITEMS.include?(orig_item_name) - names << "\"T#{index}.#{orig_item_name}\"" + add_column.call("T#{index}.#{orig_item_name}") current_position += 1 next end @@ -1044,11 +1075,11 @@ def self.tsdb_lookup_single_db_shard(items, start_time:, end_time: nil, scope: " suffix = column_suffix_for_value_type(value_type) col_name = "T#{index}.#{safe_item_name}#{suffix}" - names << "\"#{col_name}\"" + add_column.call(col_name) item_types[col_name] = resolve_item_type(item_def, value_type) current_position += 1 if limits - names << "\"T#{index}.#{safe_item_name}__L\"" + add_column.call("T#{index}.#{safe_item_name}__L") end end @@ -1094,7 +1125,7 @@ def self.tsdb_lookup_single_db_shard(items, start_time:, end_time: nil, scope: " row_index = 0 row_timestamps = {} tuples.each do |tuple| - col_name = tuple[0] + col_name = aliases.fetch(tuple[0], tuple[0]) col_value = tuple[1] if col_name.include?("__L") data[row_num][row_index - 1][1] = col_value diff --git a/openc3/python/openc3/api/limits_api.py b/openc3/python/openc3/api/limits_api.py index 47a56ed4f7..68f4d3368a 100644 --- a/openc3/python/openc3/api/limits_api.py +++ b/openc3/python/openc3/api/limits_api.py @@ -12,7 +12,7 @@ from datetime import datetime, timezone from openc3.api import WHITELIST -from openc3.api.tlm_api import _tlm_process_args +from openc3.api.tlm_api import _tlm_packet_names, _tlm_process_args, _tlm_process_args_all_packets from openc3.environment import OPENC3_SCOPE from openc3.models.cvt_model import CvtModel from openc3.models.target_model import TargetModel @@ -128,40 +128,14 @@ def limits_enabled(*args, scope=OPENC3_SCOPE): # # Favor the first syntax where possible as it is more succinct. # -# @param args [String|Array] See the description for calling style +# @param args [String|Array] See the description for calling style. +# A packet name of LATEST applies to every packet which contains the item. def enable_limits(*args, scope=OPENC3_SCOPE): - target_name, packet_name, item_name = _tlm_process_args(args, "enable_limits", scope=scope) - authorize( - permission="tlm_set", - target_name=target_name, - packet_name=packet_name, - scope=scope, - ) - packet = TargetModel.packet(target_name, packet_name, scope=scope) - found_item = None - for item in packet["items"]: - if item["name"] == item_name: - item["limits"]["enabled"] = True - found_item = item - break - if found_item is None: - raise RuntimeError(f"Item '{target_name} {packet_name} {item_name}' does not exist while enabling limits") - - TargetModel.set_packet(target_name, packet_name, packet, scope=scope) - - message = f"Enabling Limits For '{target_name} {packet_name} {item_name}'" - Logger.info(message, scope=scope) - - event = { - "type": "LIMITS_ENABLE_STATE", - "target_name": target_name, - "packet_name": packet_name, - "item_name": item_name, - "enabled": True, - "time_nsec": to_nsec_from_epoch(datetime.now(timezone.utc)), - "message": message, - } - LimitsEventTopic.write(event, scope=scope) + target_name, packet_names, item_name = _tlm_process_args_all_packets(args, "enable_limits", scope=scope) + for packet_name in packet_names: + authorize(permission="tlm_set", target_name=target_name, packet_name=packet_name, scope=scope) + for packet_name in packet_names: + _set_limits_enabled(target_name, packet_name, item_name, True, scope) # Disable limit checking for a telemetry item @@ -172,40 +146,14 @@ def enable_limits(*args, scope=OPENC3_SCOPE): # # Favor the first syntax where possible as it is more succinct. # -# @param args [String|Array] See the description for calling style +# @param args [String|Array] See the description for calling style. +# A packet name of LATEST applies to every packet which contains the item. def disable_limits(*args, scope=OPENC3_SCOPE): - target_name, packet_name, item_name = _tlm_process_args(args, "disable_limits", scope=scope) - authorize( - permission="tlm_set", - target_name=target_name, - packet_name=packet_name, - scope=scope, - ) - packet = TargetModel.packet(target_name, packet_name, scope=scope) - found_item = None - for item in packet["items"]: - if item["name"] == item_name: - item["limits"].pop("enabled", None) - found_item = item - break - if found_item is None: - raise RuntimeError(f"Item '{target_name} {packet_name} {item_name}' does not exist while disabling limits") - - TargetModel.set_packet(target_name, packet_name, packet, scope=scope) - - message = f"Disabling Limits for '{target_name} {packet_name} {item_name}'" - Logger.info(message, scope=scope) - - event = { - "type": "LIMITS_ENABLE_STATE", - "target_name": target_name, - "packet_name": packet_name, - "item_name": item_name, - "enabled": False, - "time_nsec": to_nsec_from_epoch(datetime.now(timezone.utc)), - "message": message, - } - LimitsEventTopic.write(event, scope=scope) + target_name, packet_names, item_name = _tlm_process_args_all_packets(args, "disable_limits", scope=scope) + for packet_name in packet_names: + authorize(permission="tlm_set", target_name=target_name, packet_name=packet_name, scope=scope) + for packet_name in packet_names: + _set_limits_enabled(target_name, packet_name, item_name, False, scope) # Get a Hash of all the limits sets defined for an item. Hash keys are the limit @@ -218,9 +166,15 @@ def disable_limits(*args, scope=OPENC3_SCOPE): # # @return [Hash{String => Array}] def get_limits(target_name, packet_name, item_name, scope=OPENC3_SCOPE): + target_name = target_name.upper() + packet_name = packet_name.upper() + item_name = item_name.upper() + # Resolve LATEST before authorizing so the real packet is authorized + if packet_name == "LATEST": + packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, scope=scope) authorize(permission="tlm", target_name=target_name, packet_name=packet_name, scope=scope) limits = {} - item = _get_item(target_name, packet_name, item_name, scope=scope) + item = TargetModel.packet_item(target_name, packet_name, item_name, scope=scope) for key, vals in item["limits"].items(): if not isinstance(vals, dict): continue @@ -237,7 +191,8 @@ def get_limits(target_name, packet_name, item_name, scope=OPENC3_SCOPE): # Change the limits settings for a given item. By default, a new limits set called 'CUSTOM' -# is created to avoid overriding existing limits. +# is created to avoid overriding existing limits. A packet name of LATEST +# applies to every packet which contains the item. def set_limits( target_name, packet_name, @@ -253,73 +208,76 @@ def set_limits( enabled=True, scope=OPENC3_SCOPE, ): - authorize( - permission="tlm_set", - target_name=target_name, - packet_name=packet_name, - scope=scope, + target_name = target_name.upper() + packet_name = packet_name.upper() + item_name = item_name.upper() + packet_names = ( + _tlm_packet_names(target_name, packet_name, item_name, scope) if packet_name == "LATEST" else [packet_name] ) + for name in packet_names: + authorize(permission="tlm_set", target_name=target_name, packet_name=name, scope=scope) if (red_low > yellow_low) or (yellow_low >= yellow_high) or (yellow_high > red_high): raise RuntimeError("Invalid limits specified. Ensure yellow limits are within red limits.") if (green_low is not None and green_high is not None) and ( (yellow_low > green_low) or (green_low >= green_high) or (green_high > yellow_high) ): raise RuntimeError("Invalid limits specified. Ensure green limits are within yellow limits.") - packet = TargetModel.packet(target_name, packet_name, scope=scope) - found_item = None - for item in packet["items"]: - if item["name"] == item_name: - if item["limits"]: - if persistence: - item["limits"]["persistence_setting"] = persistence - if enabled: - item["limits"]["enabled"] = True - else: - item["limits"].pop("enabled", None) - limits = {} - limits["red_low"] = red_low - limits["yellow_low"] = yellow_low - limits["yellow_high"] = yellow_high - limits["red_high"] = red_high - if green_low is not None and green_high is not None: - limits["green_low"] = green_low - if green_low is not None and green_high is not None: - limits["green_high"] = green_high - item["limits"][limits_set] = limits - found_item = item - break - else: - raise RuntimeError("Cannot set_limits on item without any limits") - if found_item is None: - raise RuntimeError(f"Item '{target_name} {packet_name} {item_name}' does not exist while setting limits") - message = ( - f"Setting '{target_name} {packet_name} {item_name}' limits to {red_low} {yellow_low} {yellow_high} {red_high}" - ) - if green_low is not None and green_high is not None: - message += f" {green_low} {green_high}" - message += f" in set {limits_set} with persistence {persistence} as enabled {enabled}" - Logger.info(message, scope=scope) - - TargetModel.set_packet(target_name, packet_name, packet, scope=scope) - - event = { - "type": "LIMITS_SETTINGS", - "target_name": target_name, - "packet_name": packet_name, - "item_name": item_name, - "red_low": red_low, - "yellow_low": yellow_low, - "yellow_high": yellow_high, - "red_high": red_high, - "green_low": green_low, - "green_high": green_high, - "limits_set": limits_set, - "persistence": persistence, - "enabled": enabled, - "time_nsec": to_nsec_from_epoch(datetime.now(timezone.utc)), - "message": message, - } - LimitsEventTopic.write(event, scope=scope) + # Validate every packet before changing any so an error doesn't leave a partial update. + # TargetModel.packet returns a cached dict so it must not be modified until validated. + items = [] + for name in packet_names: + packet = TargetModel.packet(target_name, name, scope=scope) + item = next((packet_item for packet_item in packet["items"] if packet_item["name"] == item_name), None) + if item is None: + raise RuntimeError(f"Item '{target_name} {name} {item_name}' does not exist while setting limits") + if not item["limits"]: + raise RuntimeError("Cannot set_limits on item without any limits") + items.append((name, packet, item)) + for name, packet, item in items: + if persistence: + item["limits"]["persistence_setting"] = persistence + if enabled: + item["limits"]["enabled"] = True + else: + item["limits"].pop("enabled", None) + limits = {} + limits["red_low"] = red_low + limits["yellow_low"] = yellow_low + limits["yellow_high"] = yellow_high + limits["red_high"] = red_high + if green_low is not None and green_high is not None: + limits["green_low"] = green_low + limits["green_high"] = green_high + item["limits"][limits_set] = limits + + message = ( + f"Setting '{target_name} {name} {item_name}' limits to {red_low} {yellow_low} {yellow_high} {red_high}" + ) + if green_low is not None and green_high is not None: + message += f" {green_low} {green_high}" + message += f" in set {limits_set} with persistence {persistence} as enabled {enabled}" + Logger.info(message, scope=scope) + + TargetModel.set_packet(target_name, name, packet, scope=scope) + + event = { + "type": "LIMITS_SETTINGS", + "target_name": target_name, + "packet_name": name, + "item_name": item_name, + "red_low": red_low, + "yellow_low": yellow_low, + "yellow_high": yellow_high, + "red_high": red_high, + "green_low": green_low, + "green_high": green_high, + "limits_set": limits_set, + "persistence": persistence, + "enabled": enabled, + "time_nsec": to_nsec_from_epoch(datetime.now(timezone.utc)), + "message": message, + } + LimitsEventTopic.write(event, scope=scope) # Change the color associated with a telemetry item state in realtime. @@ -331,13 +289,16 @@ def set_limits( # @param item_name [str] Item name # @param state_name [str] Name of the state to change (e.g. 'CONNECTED') # @param color [str] New color for the state. Must be GREEN, YELLOW, or RED. +# A packet name of LATEST applies to every packet which contains the item. def set_state_color(target_name, packet_name, item_name, state_name, color, scope=OPENC3_SCOPE): - authorize( - permission="tlm_set", - target_name=target_name, - packet_name=packet_name, - scope=scope, + target_name = target_name.upper() + packet_name = packet_name.upper() + item_name = item_name.upper() + packet_names = ( + _tlm_packet_names(target_name, packet_name, item_name, scope) if packet_name == "LATEST" else [packet_name] ) + for name in packet_names: + authorize(permission="tlm_set", target_name=target_name, packet_name=name, scope=scope) state_name = str(state_name).upper() if color is not None: color = str(color).upper() @@ -345,46 +306,47 @@ def set_state_color(target_name, packet_name, item_name, state_name, color, scop raise RuntimeError( f"Invalid state color '{color}'. Must be one of {', '.join(PacketItem.VALID_STATE_COLORS)}." ) - packet = TargetModel.packet(target_name, packet_name, scope=scope) - found_item = None - for item in packet["items"]: - if item["name"] == item_name: - if not (item.get("states") and item["states"].get(state_name)): - raise RuntimeError( - f"State '{state_name}' does not exist for item '{target_name} {packet_name} {item_name}'" - ) - if color is None: - # Clear the state color - item["states"][state_name].pop("color", None) - else: - item["states"][state_name]["color"] = color - # Defining a state color implies limits are enabled for the item - if not item.get("limits"): - item["limits"] = {} - item["limits"]["enabled"] = True - found_item = item - break - if found_item is None: - raise RuntimeError(f"Item '{target_name} {packet_name} {item_name}' does not exist while setting state color") - if color is None: - message = f"Clearing '{target_name} {packet_name} {item_name}' state {state_name} color" - else: - message = f"Setting '{target_name} {packet_name} {item_name}' state {state_name} color to {color}" - Logger.info(message, scope=scope) - - TargetModel.set_packet(target_name, packet_name, packet, scope=scope) + # Validate every packet before changing any so an error doesn't leave a partial update. + # TargetModel.packet returns a cached dict so it must not be modified until validated. + items = [] + for name in packet_names: + packet = TargetModel.packet(target_name, name, scope=scope) + item = next((packet_item for packet_item in packet["items"] if packet_item["name"] == item_name), None) + if item is None: + raise RuntimeError(f"Item '{target_name} {name} {item_name}' does not exist while setting state color") + if not (item.get("states") and item["states"].get(state_name)): + raise RuntimeError(f"State '{state_name}' does not exist for item '{target_name} {name} {item_name}'") + items.append((name, packet, item)) + for name, packet, item in items: + if color is None: + # Clear the state color + item["states"][state_name].pop("color", None) + else: + item["states"][state_name]["color"] = color + # Defining a state color implies limits are enabled for the item + if not item.get("limits"): + item["limits"] = {} + item["limits"]["enabled"] = True - event = { - "type": "LIMITS_STATE_COLOR", - "target_name": target_name, - "packet_name": packet_name, - "item_name": item_name, - "state_name": state_name, - "color": color, - "time_nsec": to_nsec_from_epoch(datetime.now(timezone.utc)), - "message": message, - } - LimitsEventTopic.write(event, scope=scope) + if color is None: + message = f"Clearing '{target_name} {name} {item_name}' state {state_name} color" + else: + message = f"Setting '{target_name} {name} {item_name}' state {state_name} color to {color}" + Logger.info(message, scope=scope) + + TargetModel.set_packet(target_name, name, packet, scope=scope) + + event = { + "type": "LIMITS_STATE_COLOR", + "target_name": target_name, + "packet_name": name, + "item_name": item_name, + "state_name": state_name, + "color": color, + "time_nsec": to_nsec_from_epoch(datetime.now(timezone.utc)), + "message": message, + } + LimitsEventTopic.write(event, scope=scope) # Returns all limits_groups and their members @@ -531,15 +493,32 @@ def _limits_group(group_name, action, scope): TargetModel.set_packet(last_target_name, last_packet_name, packet, scope=scope) -# Gets an item. The code below is mostly duplicated from tlm_process_args in tlm_api.rb. -# -# @param target_name [String] target name -# @param packet_name [String] packet name -# @param item_name [String] item name -# @param scope [String] scope -# @return Hash The requested item based on the packet name -def _get_item(target_name, packet_name, item_name, cache_timeout=0.1, scope=OPENC3_SCOPE): - # Determine if this item exists, it will raise appropriate errors if not - if packet_name == "LATEST": - packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout, scope) - return TargetModel.packet_item(target_name, packet_name, item_name, scope=scope) +def _set_limits_enabled(target_name, packet_name, item_name, enabled, scope): + packet = TargetModel.packet(target_name, packet_name, scope=scope) + item = next((packet_item for packet_item in packet["items"] if packet_item["name"] == item_name), None) + if item is None: + action = "enabling" if enabled else "disabling" + raise RuntimeError(f"Item '{target_name} {packet_name} {item_name}' does not exist while {action} limits") + + if enabled: + item["limits"]["enabled"] = True + else: + item["limits"].pop("enabled", None) + TargetModel.set_packet(target_name, packet_name, packet, scope=scope) + + if enabled: + message = f"Enabling Limits For '{target_name} {packet_name} {item_name}'" + else: + message = f"Disabling Limits for '{target_name} {packet_name} {item_name}'" + Logger.info(message, scope=scope) + + event = { + "type": "LIMITS_ENABLE_STATE", + "target_name": target_name, + "packet_name": packet_name, + "item_name": item_name, + "enabled": enabled, + "time_nsec": to_nsec_from_epoch(datetime.now(timezone.utc)), + "message": message, + } + LimitsEventTopic.write(event, scope=scope) diff --git a/openc3/python/openc3/api/tlm_api.py b/openc3/python/openc3/api/tlm_api.py index 542e3c3aa2..91fd64b6fb 100644 --- a/openc3/python/openc3/api/tlm_api.py +++ b/openc3/python/openc3/api/tlm_api.py @@ -142,6 +142,7 @@ def inject_tlm( scope=OPENC3_SCOPE, received_time=None, ): + _tlm_reject_latest(str(packet_name).upper(), "inject_tlm") authorize( permission="tlm_set", target_name=target_name, @@ -212,17 +213,15 @@ def inject_tlm( # # @param args The args must either be a string followed by a value or # three strings followed by a value (see the calling style in the -# description). +# description). A packet name of LATEST overrides the item in every +# packet which contains it. # @param type [Symbol] Telemetry type, :ALL (default), :RAW, :CONVERTED, :FORMATTED def override_tlm(*args, type="ALL", scope=OPENC3_SCOPE): - target_name, packet_name, item_name, value = _set_tlm_process_args(args, "override_tlm", scope=scope) - authorize( - permission="tlm_set", - target_name=target_name, - packet_name=packet_name, - scope=scope, - ) - CvtModel.override(target_name, packet_name, item_name, value, type=type, scope=scope) + target_name, packet_names, item_name, value = _set_tlm_process_args_all_packets(args, "override_tlm", scope=scope) + for packet_name in packet_names: + authorize(permission="tlm_set", target_name=target_name, packet_name=packet_name, scope=scope) + for packet_name in packet_names: + CvtModel.override(target_name, packet_name, item_name, value, type=type, scope=scope) # Get the list of CVT overrides @@ -241,18 +240,16 @@ def get_overrides(scope=OPENC3_SCOPE): # Favor the first syntax where possible as it is more succinct. # # @param args The args must either be a string or three strings -# (see the calling style in the description). +# (see the calling style in the description). A packet name of LATEST +# normalizes the item in every packet which contains it. # @param type [Symbol] Telemetry type, :ALL (default), :RAW, :CONVERTED, :FORMATTED # Also takes :ALL which means to normalize all telemetry types def normalize_tlm(*args, type="ALL", scope=OPENC3_SCOPE): - target_name, packet_name, item_name = _tlm_process_args(args, "normalize_tlm", scope=scope) - authorize( - permission="tlm_set", - target_name=target_name, - packet_name=packet_name, - scope=scope, - ) - CvtModel.normalize(target_name, packet_name, item_name, type=type, scope=scope) + target_name, packet_names, item_name = _tlm_process_args_all_packets(args, "normalize_tlm", scope=scope) + for packet_name in packet_names: + authorize(permission="tlm_set", target_name=target_name, packet_name=packet_name, scope=scope) + for packet_name in packet_names: + CvtModel.normalize(target_name, packet_name, item_name, type=type, scope=scope) # Returns the raw buffer for a telemetry packet. @@ -329,45 +326,36 @@ def get_tlm_available(items, manual=False, scope=OPENC3_SCOPE): packet_name = orig_packet_name if orig_packet_name == "LATEST": + # A LATEST item can come from any packet containing it. Report the best value + # type and limits of any of them since the CVT falls back to the available + # type and historical lookups check the type available in each packet. # We can have a large cache_timeout of 1 because all we're trying to do is lookup a packet packet_name = CvtModel.determine_latest_packet_for_item( target_name, item_name, cache_timeout=1, scope=scope ) - - authorize( - permission="tlm", - target_name=target_name, - packet_name=packet_name, - scope=scope, - ) + packet_names = _tlm_packet_names(target_name, orig_packet_name, item_name, scope) + else: + packet_names = [packet_name] + + # Authorize every packet whose definition is read below, not just the latest one + for name in dict.fromkeys([*packet_names, packet_name]): + authorize( + permission="tlm", + target_name=target_name, + packet_name=name, + scope=scope, + ) try: - item_config = TargetModel.packet_item(target_name, packet_name, item_name, scope=scope) - if item_name in Packet.RESERVED_ITEM_NAMES: - value_type = "RAW" # Must request the raw value when dealing with the reserved items - - # Arrays must be accessed as RAW since there's no conversion - if item_config.get("array_size") is not None: - value_type = "RAW" - - # Determine the best available value type based on item configuration - if value_type == "FORMATTED" or value_type == "WITH_UNITS": - if item_config.get("format_string") or item_config.get("units"): - result_item = "__".join([target_name, orig_packet_name, item_name, "FORMATTED"]) - elif item_config.get("read_conversion") or item_config.get("states"): - result_item = "__".join([target_name, orig_packet_name, item_name, "CONVERTED"]) - else: - result_item = "__".join([target_name, orig_packet_name, item_name, "RAW"]) - elif value_type == "CONVERTED": - if item_config.get("read_conversion") or item_config.get("states"): - result_item = "__".join([target_name, orig_packet_name, item_name, "CONVERTED"]) - else: - result_item = "__".join([target_name, orig_packet_name, item_name, "RAW"]) - else: # RAW or unknown - result_item = "__".join([target_name, orig_packet_name, item_name, "RAW"]) + item_configs = [TargetModel.packet_item(target_name, name, item_name, scope=scope) for name in packet_names] + available_type = max( + (_tlm_available_value_type(item_config, item_name, value_type) for item_config in item_configs), + key=VALUE_TYPE_ORDER.index, + ) + result_item = "__".join([target_name, orig_packet_name, item_name, available_type]) # Tack on __LIMITS to notify that we have an available limits value - if item_config.get("limits", {}).get("DEFAULT"): + if any(item_config.get("limits", {}).get("DEFAULT") for item_config in item_configs): result_item += "__LIMITS" results.append(result_item) @@ -398,6 +386,8 @@ def get_tlm_values( raise TypeError("items must be array of strings: ['TGT__PKT__ITEM__TYPE', ...]") packets = [] cvt_items = [] + # Index of each historical LATEST item in cvt_items => every packet containing the item + latest_candidates = {} for item in items: # get_tlm_available returns None for items which don't exist and its result is # passed directly here, so None is a placeholder which returns a None value @@ -414,8 +404,21 @@ def get_tlm_values( target_name, packet_name, item_name, value_type = parts[0:4] limits = parts[4] if len(parts) == 5 else None if packet_name == "LATEST": + if start_time is not None: + # The packet received most recently now isn't necessarily the one received most + # recently at start_time so keep every candidate packet for _tlm_latest_history + # Each packet must be queried for the value type and limits it has or the query fails + candidates = [] + for candidate in _tlm_packet_names(target_name, packet_name, item_name, scope): + packets.append([target_name, candidate]) + candidate_item = TargetModel.packet_item(target_name, candidate, item_name, scope=scope) + candidate_limits = limits if limits and candidate_item.get("limits", {}).get("DEFAULT") else None + candidate_type = _tlm_available_value_type(candidate_item, item_name, value_type) + candidates.append(TlmItem(target_name, candidate, item_name, candidate_type, candidate_limits)) + latest_candidates[len(cvt_items)] = candidates + cvt_items.append(TlmItem(target_name, packet_name, item_name, value_type, limits)) + continue packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout, scope) - # Change packet_name in case of LATEST and ensure upcase # NOTE: limits is required by the historical (start_time) QuestDB lookup cvt_items.append(TlmItem(target_name, packet_name, item_name, value_type, limits)) packets.append([target_name, packet_name]) @@ -428,6 +431,8 @@ def get_tlm_values( packet_name=name[1], scope=scope, ) + if latest_candidates: + return _tlm_latest_history(cvt_items, latest_candidates, start_time, end_time, cache_timeout, scope) return CvtModel.get_tlm_values(cvt_items, stale_time, cache_timeout, start_time, end_time, scope) @@ -545,6 +550,7 @@ def subscribe_packets(packets, scope=OPENC3_SCOPE): for target_name, packet_name in packets: target_name = target_name.upper() packet_name = packet_name.upper() + _tlm_reject_latest(packet_name, "subscribe_packets") authorize( permission="tlm", target_name=target_name, @@ -683,9 +689,16 @@ def _extract_target_packet_names(method_name, *args): raise RuntimeError( f'ERROR: Both target name and packet name required. Usage: {method_name}("TGT PKT") or {method_name}("TGT", "PKT")' ) + _tlm_reject_latest(packet_name, method_name) return target_name, packet_name +# LATEST picks a packet per item so it can't identify a whole packet +def _tlm_reject_latest(packet_name, method_name): + if packet_name == "LATEST": + raise RuntimeError(f"ERROR: LATEST is not supported by {method_name}(), specify a packet name") + + def _extract_target_packet_item_names(method_name, *args): target_name = None packet_name = None @@ -721,6 +734,31 @@ def _validate_tlm_type(tlm_type): def _tlm_process_args(args, method_name, cache_timeout=0.1, scope=OPENC3_SCOPE): + target_name, packet_name, item_name = _tlm_extract_args(args, method_name) + packet_name = _tlm_resolve_packet_name(target_name, packet_name, item_name, cache_timeout, scope) + return target_name, packet_name, item_name + + +def _set_tlm_process_args(args, method_name, cache_timeout=0.1, scope=OPENC3_SCOPE): + target_name, packet_name, item_name, value = _set_tlm_extract_args(args, method_name) + packet_name = _tlm_resolve_packet_name(target_name, packet_name, item_name, cache_timeout, scope) + return target_name, packet_name, item_name, value + + +# Like _tlm_process_args but returns a list of packet names where LATEST +# expands to every packet containing the item. Used by APIs which change +# an item so the change applies wherever the item appears. +def _tlm_process_args_all_packets(args, method_name, scope=OPENC3_SCOPE): + target_name, packet_name, item_name = _tlm_extract_args(args, method_name) + return target_name, _tlm_packet_names(target_name, packet_name, item_name, scope), item_name + + +def _set_tlm_process_args_all_packets(args, method_name, scope=OPENC3_SCOPE): + target_name, packet_name, item_name, value = _set_tlm_extract_args(args, method_name) + return target_name, _tlm_packet_names(target_name, packet_name, item_name, scope), item_name, value + + +def _tlm_extract_args(args, method_name): match len(args): case 1: target_name, packet_name, item_name = extract_fields_from_tlm_text(args[0]) @@ -731,18 +769,10 @@ def _tlm_process_args(args, method_name, cache_timeout=0.1, scope=OPENC3_SCOPE): case _: # Invalid number of arguments raise RuntimeError(f"ERROR: Invalid number of arguments ({len(args)}) passed to {method_name}()") - target_name = target_name.upper() - packet_name = packet_name.upper() - item_name = item_name.upper() - if packet_name == "LATEST": - packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout, scope) - else: - # Determine if this item exists, it will raise appropriate errors if not - TargetModel.packet_item(target_name, packet_name, item_name, scope=scope) - return target_name, packet_name, item_name + return target_name.upper(), packet_name.upper(), item_name.upper() -def _set_tlm_process_args(args, method_name, cache_timeout=0.1, scope=OPENC3_SCOPE): +def _set_tlm_extract_args(args, method_name): match len(args): case 1: target_name, packet_name, item_name, value = extract_fields_from_set_tlm_text(args[0]) @@ -754,13 +784,175 @@ def _set_tlm_process_args(args, method_name, cache_timeout=0.1, scope=OPENC3_SCO case _: # Invalid number of arguments raise RuntimeError(f"ERROR: Invalid number of arguments ({len(args)}) passed to {method_name}()") - target_name = target_name.upper() - packet_name = packet_name.upper() - item_name = item_name.upper() + return target_name.upper(), packet_name.upper(), item_name.upper(), value + + +# Resolve LATEST to the most recently received packet containing the item +def _tlm_resolve_packet_name(target_name, packet_name, item_name, cache_timeout=0.1, scope=OPENC3_SCOPE): if packet_name == "LATEST": - packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout, scope) - else: - # Determine if this item exists, it will raise appropriate errors if not - TargetModel.packet_item(target_name, packet_name, item_name, scope=scope) + return CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout, scope) + # Determine if this item exists, it will raise appropriate errors if not + TargetModel.packet_item(target_name, packet_name, item_name, scope=scope) + return packet_name + + +VALUE_TYPE_ORDER = ["RAW", "CONVERTED", "FORMATTED"] + + +# Returns the value type available for an item given the requested value type +def _tlm_available_value_type(item_config, item_name, value_type): + # Must request the raw value when dealing with the reserved items + if item_name in Packet.RESERVED_ITEM_NAMES: + return "RAW" + # Arrays must be accessed as RAW since there's no conversion + if item_config.get("array_size") is not None: + return "RAW" + # This logic must match the logic in Packet.decom + converted = item_config.get("states") or ( + item_config.get("read_conversion") and item_config.get("data_type") != "DERIVED" + ) + if value_type == "FORMATTED" or value_type == "WITH_UNITS": + if item_config.get("format_string") or item_config.get("units"): + return "FORMATTED" + return "CONVERTED" if converted else "RAW" + if value_type == "CONVERTED": + return "CONVERTED" if converted else "RAW" + return "RAW" # RAW or unknown + + +# Historical lookup where some items are LATEST. latest_candidates maps the index +# of each LATEST item in cvt_items to the item to look up in every packet which contains it. +def _tlm_latest_history(cvt_items, latest_candidates, start_time, end_time, cache_timeout, scope): + regular = [index for index in range(len(cvt_items)) if index not in latest_candidates] + + # The QuestDB lookup ASOF JOINs every packet to the rows of the first, so + # a range needs a regular item first to drive the rows. Without one keep + # the realtime behavior of using the packet received most recently now. + if end_time is not None and all(cvt_items[index].item_name is None for index in regular): + resolved = [] + for index, cvt_item in enumerate(cvt_items): + if cvt_item.packet_name == "LATEST": + packet_name = CvtModel.determine_latest_packet_for_item( + cvt_item.target_name, cvt_item.item_name, cache_timeout, scope + ) + # Use the candidate so the query asks for the value type and limits that packet has + cvt_item = next( + (candidate for candidate in latest_candidates[index] if candidate.packet_name == packet_name), + cvt_item._replace(packet_name=packet_name), + ) + resolved.append(cvt_item) + return CvtModel.tsdb_lookup(resolved, start_time=start_time, end_time=end_time, scope=scope) + + # A packet without a table fails the query so skip those candidates + packets = list( + dict.fromkeys( + (candidate.target_name, candidate.packet_name) + for candidates in latest_candidates.values() + for candidate in candidates + ) + ) + existing = CvtModel.tsdb_packets_with_tables(packets, scope=scope) + latest_candidates = { + index: [candidate for candidate in candidates if (candidate.target_name, candidate.packet_name) in existing] + for index, candidates in latest_candidates.items() + } + + if end_time is not None: + query = [] + query_index = {} + + def _add_query_item(query_item): + if query_item not in query_index: + query_index[query_item] = len(query) + query.append(query_item) + return query_index[query_item] + + # Regular items go in the query first so the first packet drives the rows + positions: dict[int, int | list[tuple[int, int]]] = {} + for index in regular: + positions[index] = _add_query_item(cvt_items[index]) + for index, candidates in latest_candidates.items(): + positions[index] = [ + ( + _add_query_item(candidate), + _add_query_item( + TlmItem(candidate.target_name, candidate.packet_name, "PACKET_TIMESECONDS", "RAW", None) + ), + ) + for candidate in candidates + ] + rows = CvtModel.tsdb_lookup(query, start_time=start_time, end_time=end_time, scope=scope, flatten=False) + if not isinstance(rows, list) or len(rows) == 0: + return rows + collapsed = [] + for row in rows: + values = [] + for index in range(len(cvt_items)): + position = positions[index] + if isinstance(position, int): + values.append(row[position]) + else: + # Take the candidate with the newest packet time at this row + newest = _tlm_newest( + [(row[time_index][0], row[value_index]) for value_index, time_index in position] + ) + values.append(newest or [None, None]) + collapsed.append(values) + # Match tsdb_lookup which returns a flat array for a single row + return collapsed[0] if len(collapsed) == 1 else collapsed + + # A point in time returns the last row of the first packet before start_time + # with the other packets ASOF JOINed at that row. A candidate packet received + # after that row would be missed so query each candidate packet on its own. + found = False + row = [[None, None] for _ in cvt_items] + if not all(cvt_items[index].item_name is None for index in regular): + result = CvtModel.tsdb_lookup( + [cvt_items[index] for index in regular], start_time=start_time, scope=scope, flatten=False + ) + if isinstance(result, list) and len(result) > 0: + found = True + for result_index, index in enumerate(regular): + row[index] = result[0][result_index] + # Group the LATEST items by candidate packet so each packet is queried once + by_packet = {} + for index, candidates in latest_candidates.items(): + for candidate in candidates: + by_packet.setdefault((candidate.target_name, candidate.packet_name), {})[index] = candidate + found_values = {} + for (target_name, packet_name), packet_items in by_packet.items(): + query = list(dict.fromkeys(packet_items.values())) + query.append(TlmItem(target_name, packet_name, "PACKET_TIMESECONDS", "RAW", None)) + result = CvtModel.tsdb_lookup(query, start_time=start_time, scope=scope, flatten=False) + if not isinstance(result, list) or len(result) == 0: + continue + values = result[0] + for index, query_item in packet_items.items(): + found_values.setdefault(index, []).append((values[-1][0], values[query.index(query_item)])) + for index, candidates in found_values.items(): + value = _tlm_newest(candidates) + if value is None: + continue + found = True + row[index] = value + return row if found else {} - return target_name, packet_name, item_name, value + +# Return the value with the newest time from (time, value) pairs, or None +def _tlm_newest(candidates): + candidates = [candidate for candidate in candidates if candidate[0] is not None] + if not candidates: + return None + return max(candidates, key=lambda candidate: candidate[0])[1] + + +# Expand LATEST to every packet containing the item +def _tlm_packet_names(target_name, packet_name, item_name, scope=OPENC3_SCOPE): + if packet_name == "LATEST": + packet_names = TargetModel.get_item_to_packet_map(target_name, scope=scope).get(item_name) + if packet_names is None: + raise RuntimeError(f"Item '{target_name} LATEST {item_name}' does not exist for scope: {scope}") + return packet_names + # Determine if this item exists, it will raise appropriate errors if not + TargetModel.packet_item(target_name, packet_name, item_name, scope=scope) + return [packet_name] diff --git a/openc3/python/openc3/models/cvt_model.py b/openc3/python/openc3/models/cvt_model.py index 5a63b466f3..59a52b098e 100644 --- a/openc3/python/openc3/models/cvt_model.py +++ b/openc3/python/openc3/models/cvt_model.py @@ -192,9 +192,15 @@ def tsdb_lookup( start_time: str, end_time: str | None = None, scope: str = OPENC3_SCOPE, + flatten: bool = True, ): """Query historical telemetry data from TSDB""" - return QuestDBClient.tsdb_lookup(items, start_time=start_time, end_time=end_time, scope=scope) + return QuestDBClient.tsdb_lookup(items, start_time=start_time, end_time=end_time, scope=scope, flatten=flatten) + + @classmethod + def tsdb_packets_with_tables(cls, packets: list, scope: str = OPENC3_SCOPE): + """Return the (target_name, packet_name) telemetry packets which have a TSDB table""" + return QuestDBClient.tlm_packets_with_tables(packets, scope=scope) # Return all item values and limit state from the CVT # diff --git a/openc3/python/openc3/models/target_model.py b/openc3/python/openc3/models/target_model.py index 56de6527b8..aac0932fce 100644 --- a/openc3/python/openc3/models/target_model.py +++ b/openc3/python/openc3/models/target_model.py @@ -263,12 +263,15 @@ def limits_groups(cls, scope: str = OPENC3_SCOPE): @classmethod def get_item_to_packet_map(cls, target_name: str, scope: str = OPENC3_SCOPE): - if target_name in TargetModel.item_map_cache: - cache_time, item_map = TargetModel.item_map_cache[target_name] + # Normalize before building the key so lowercase lookups share the entry + # that update_store_item_map refreshes + target_name = target_name.upper() + # The cache is keyed by item_map_key since the same target can exist in multiple scopes + item_map_key = f"{scope}__{target_name}__item_to_packet_map" + if item_map_key in TargetModel.item_map_cache: + cache_time, item_map = TargetModel.item_map_cache[item_map_key] if (time.time() - cache_time) < TargetModel.ITEM_MAP_CACHE_TIMEOUT: return item_map - item_map_key = f"{scope}__{target_name}__item_to_packet_map" - target_name = target_name.upper() store = cls._store_for_target(target_name, scope) json_data = store.get(item_map_key) if json_data: @@ -276,7 +279,7 @@ def get_item_to_packet_map(cls, target_name: str, scope: str = OPENC3_SCOPE): else: item_map = cls.build_item_to_packet_map(target_name, scope=scope) store.set(item_map_key, json.dumps(item_map)) - TargetModel.item_map_cache[target_name] = [time.time(), item_map] + TargetModel.item_map_cache[item_map_key] = [time.time(), item_map] return item_map @classmethod @@ -628,7 +631,7 @@ def update_store_item_map(self): item_map_key = f"{self.scope}__{self.name}__item_to_packet_map" item_map = TargetModel.build_item_to_packet_map(self.name, scope=self.scope) Store.instance(db_shard=self.db_shard).set(item_map_key, json.dumps(item_map)) - TargetModel.item_map_cache[self.name] = [time.time(), item_map] + TargetModel.item_map_cache[item_map_key] = [time.time(), item_map] def dynamic_update(self, packets, cmd_or_tlm="TELEMETRY", filename="dynamic_tlm.txt"): # Build hash of targets/packets diff --git a/openc3/python/openc3/script/api_shared.py b/openc3/python/openc3/script/api_shared.py index 7919d5ea37..2b17fe8d5d 100644 --- a/openc3/python/openc3/script/api_shared.py +++ b/openc3/python/openc3/script/api_shared.py @@ -695,6 +695,9 @@ def _wait_packet( type = "CHECK" else: type = "WAIT" + # LATEST would compare the RECEIVED_COUNT of whichever packet was received most recently + if str(packet_name).upper() == "LATEST": + raise RuntimeError("LATEST is not supported by wait_packet, specify a packet name") initial_count = openc3.script.API_SERVER.tlm(target_name, packet_name, "RECEIVED_COUNT", scope=scope) # If the packet has not been received the initial_count could be None if initial_count is None: diff --git a/openc3/python/openc3/utilities/questdb_client.py b/openc3/python/openc3/utilities/questdb_client.py index 0326c6d0d4..fce81d4dbb 100644 --- a/openc3/python/openc3/utilities/questdb_client.py +++ b/openc3/python/openc3/utilities/questdb_client.py @@ -805,7 +805,33 @@ def db_shard_for_target(cls, target_name, scope="DEFAULT"): return Store.db_shard_for_target(target_name, scope=scope) @classmethod - def tsdb_lookup(cls, items, start_time, end_time=None, scope="DEFAULT"): + def tlm_packets_with_tables(cls, packets, scope="DEFAULT"): + """Return the telemetry packets which have a table. The TSDB microservice only + creates tables for its initial packets so a packet added later has no table + until it's received. If the table names can't be fetched assume they all exist. + + Args: + packets: Iterable of (target_name, packet_name) pairs + scope: Scope name + + Returns: + Set of the (target_name, packet_name) pairs which have a table + """ + by_db_shard = {} + for target_name, packet_name in packets: + db_shard = cls.db_shard_for_target(target_name, scope=scope) + by_db_shard.setdefault(db_shard, []).append((target_name, packet_name)) + existing = set() + for db_shard, shard_packets in by_db_shard.items(): + names = cls.table_names(db_shard=db_shard) + for target_name, packet_name in shard_packets: + table_name, _ = cls.sanitize_table_name(target_name, packet_name, scope=scope) + if names is None or table_name.upper() in names: + existing.add((target_name, packet_name)) + return existing + + @classmethod + def tsdb_lookup(cls, items, start_time, end_time=None, scope="DEFAULT", flatten=True): """Query historical telemetry data from QuestDB for a list of items. Builds the SQL query, executes it, and decodes all results. Supports cross-db_shard queries by grouping items by db_shard, executing @@ -817,17 +843,20 @@ def tsdb_lookup(cls, items, start_time, end_time=None, scope="DEFAULT"): start_time: Start timestamp for the query end_time: End timestamp, or None for "latest single row" scope: Scope name + flatten: Whether a single row result is returned as a flat array Returns: Array of [value, limits_state] pairs per row, or {} if no results. - Single-row results return a flat array; multi-row results return array of arrays. + Single-row results return a flat array (unless flatten is False); multi-row results + return array of arrays. """ items = [TlmItem(*item) for item in items] # Every item is a placeholder for an item which doesn't exist, so there's # nothing to query. Return a single row of None values, one per item. if all(item.item_name is None for item in items): - return [[None, None] for _ in items] + row = [[None, None] for _ in items] + return row if flatten else [row] # Group items by db_shard number while preserving their original positions db_shard_groups = {} # db_shard => {"positions": [], "items": []} @@ -841,7 +870,12 @@ def tsdb_lookup(cls, items, start_time, end_time=None, scope="DEFAULT"): if len(db_shard_groups) == 1: db_shard, group = next(iter(db_shard_groups.items())) return cls._tsdb_lookup_single_db_shard( - group["items"], start_time=start_time, end_time=end_time, scope=scope, db_shard=db_shard + group["items"], + start_time=start_time, + end_time=end_time, + scope=scope, + db_shard=db_shard, + flatten=flatten, ) # Cross-db_shard: execute per-db_shard queries (as lists of rows) and merge results @@ -866,7 +900,7 @@ def tsdb_lookup(cls, items, start_time, end_time=None, scope="DEFAULT"): if db_shard_idx < len(row) and row[db_shard_idx]: merged[row_num][orig_pos] = row[db_shard_idx] # Match the single db_shard behavior of returning a flat array for a single row - if max_rows == 1: + if flatten and max_rows == 1: return merged[0] return merged @@ -884,6 +918,14 @@ def _tsdb_lookup_single_db_shard(cls, items, start_time, end_time=None, scope="D calculated_items = {} needed_timestamps = {} current_position = 0 + # Columns with the same name in different tables come back renamed by QuestDB + # (e.g. VALUE and VALUE_2) so alias every column and map it back when decoding + aliases = {} + + def add_column(col_name): + alias_name = f"__col{len(aliases)}" + aliases[alias_name] = col_name + names.append(f'"{col_name}" as {alias_name}') for item in items: target_name, packet_name, item_name, value_type, limits = item @@ -898,7 +940,7 @@ def _tsdb_lookup_single_db_shard(cls, items, start_time, end_time=None, scope="D index = list(tables.keys()).index(table_name) if item_name in cls.STORED_TIMESTAMP_ITEMS: - names.append(f'"T{index}.{item_name}"') + add_column(f"T{index}.{item_name}") current_position += 1 continue @@ -926,12 +968,12 @@ def _tsdb_lookup_single_db_shard(cls, items, start_time, end_time=None, scope="D suffix = cls.column_suffix_for_value_type(value_type) col_name = f"T{index}.{safe_item_name}{suffix}" - names.append(f'"{col_name}"') + add_column(col_name) item_types[col_name] = cls.resolve_item_type(item_def, value_type) current_position += 1 if limits: - names.append(f'"T{index}.{safe_item_name}__L"') + add_column(f"T{index}.{safe_item_name}__L") # Every item in this db_shard is a placeholder so there's no table to query. # Return no results and let tsdb_lookup fill these positions with [None, None] @@ -971,7 +1013,8 @@ def _tsdb_lookup_single_db_shard(cls, items, start_time, end_time=None, scope="D data.append([]) col_index = 0 row_timestamps = {} - for col_name, col_value in row.items(): + for result_name, col_value in row.items(): + col_name = aliases.get(result_name, result_name) if "__L" in col_name: if col_index > 0: data[row_index][col_index - 1] = [ diff --git a/openc3/python/test/api/test_limits_api.py b/openc3/python/test/api/test_limits_api.py index 5a10d4860d..8e2680cb45 100644 --- a/openc3/python/test/api/test_limits_api.py +++ b/openc3/python/test/api/test_limits_api.py @@ -30,6 +30,8 @@ class TestLimitsApi(unittest.TestCase): def setUp(self, system): redis = mock_redis(self) setup_system() + # Other tests can leave a fake item map in the class level cache + TargetModel.item_map_cache = {} orig_xread = redis.xread @@ -117,6 +119,14 @@ def test_gets_limits_for_a_latest_item(self): }, ) + @patch("openc3.api.limits_api.authorize") + def test_get_limits_authorizes_the_resolved_packet_rather_than_latest(self, mock_authorize): + with patch.object(CvtModel, "determine_latest_packet_for_item", return_value="HEALTH_STATUS"): + get_limits("inst", "latest", "temp1") + mock_authorize.assert_called_once_with( + permission="tlm", target_name="INST", packet_name="HEALTH_STATUS", scope="DEFAULT" + ) + def test_set_limits_complains_about_non_existant_targets(self): with self.assertRaisesRegex(RuntimeError, "Packet definition 'BLAH HEALTH_STATUS' does not exist"): set_limits("BLAH", "HEALTH_STATUS", "TEMP1", 0.0, 10.0, 20.0, 30.0) @@ -286,6 +296,37 @@ def test_set_state_color_writes_a_limits_state_color_event_with_none_when_cleari self.assertEqual(event["state_name"], "UNAVAILABLE") self.assertIsNone(event["color"]) + def test_enables_and_disables_latest_limits_in_every_packet_which_contains_the_item(self): + packet_names = TargetModel.get_item_to_packet_map("INST", scope="DEFAULT")["CCSDSSEQCNT"] + self.assertGreater(len(packet_names), 1) + disable_limits("INST LATEST CCSDSSEQCNT") + for packet_name in packet_names: + self.assertFalse(limits_enabled("INST", packet_name, "CCSDSSEQCNT")) + enable_limits("INST", "LATEST", "CCSDSSEQCNT") + for packet_name in packet_names: + self.assertTrue(limits_enabled("INST", packet_name, "CCSDSSEQCNT")) + + def test_set_limits_through_latest(self): + set_limits("INST", "LATEST", "TEMP1", 0.0, 10.0, 20.0, 30.0) + self.assertEqual(get_limits("INST", "HEALTH_STATUS", "TEMP1")["CUSTOM"], [0.0, 10.0, 20.0, 30.0]) + + def test_set_limits_through_latest_sets_no_packet_if_any_packet_fails(self): + item_map = {"TEMP1": ["HEALTH_STATUS", "MECH"]} + with ( + patch.object(TargetModel, "get_item_to_packet_map", return_value=item_map), + self.assertRaisesRegex(RuntimeError, "Item 'INST MECH TEMP1' does not exist while setting limits"), + ): + set_limits("INST", "LATEST", "TEMP1", 0.0, 10.0, 20.0, 30.0) + self.assertNotIn("CUSTOM", get_limits("INST", "HEALTH_STATUS", "TEMP1")) + + def test_set_state_color_through_latest_in_every_packet_which_contains_the_item(self): + packet_names = TargetModel.get_item_to_packet_map("INST", scope="DEFAULT")["CCSDSTYPE"] + self.assertGreater(len(packet_names), 1) + set_state_color("INST", "LATEST", "CCSDSTYPE", "TLM", "GREEN") + for packet_name in packet_names: + item = TargetModel.packet_item("INST", packet_name, "CCSDSTYPE", scope="DEFAULT") + self.assertEqual(item["states"]["TLM"]["color"], "GREEN") + def test_get_limits_groups_returns_all_the_limits_groups(self): self.assertEqual( get_limits_groups(), diff --git a/openc3/python/test/api/test_tlm_api.py b/openc3/python/test/api/test_tlm_api.py index d349d6def1..6d2f5cba87 100644 --- a/openc3/python/test/api/test_tlm_api.py +++ b/openc3/python/test/api/test_tlm_api.py @@ -31,10 +31,38 @@ from test.test_helper import System, mock_redis, setup_system +ORIGINAL_PACKET_ITEM = TargetModel.packet_item +PLAIN_TEMP1 = {"name": "TEMP1", "data_type": "UINT", "limits": {}} + + +# Make TEMP1 a LATEST item in HEALTH_STATUS and PARAMS. PARAMS TEMP1 is the +# same as HEALTH_STATUS TEMP1 unless params_temp1 is given. +def latest_candidates(params_temp1=None): + def packet_item(target_name, packet_name, item_name, type="TLM", scope="DEFAULT"): + if packet_name == "PARAMS" and item_name == "TEMP1": + return params_temp1 or ORIGINAL_PACKET_ITEM("INST", "HEALTH_STATUS", "TEMP1", scope=scope) + return ORIGINAL_PACKET_ITEM(target_name, packet_name, item_name, type=type, scope=scope) + + def item_map(target_name, scope="DEFAULT"): + return {"TEMP1": ["HEALTH_STATUS", "PARAMS"]} + + def packets_with_tables(packets, scope="DEFAULT"): + return set(packets) + + def decorator(func): + func = patch.object(TargetModel, "packet_item", new=packet_item)(func) + func = patch.object(CvtModel, "tsdb_packets_with_tables", new=packets_with_tables)(func) + return patch.object(TargetModel, "get_item_to_packet_map", new=item_map)(func) + + return decorator + + class TestTlmApi(unittest.TestCase): def setUp(self): redis = mock_redis(self) setup_system() + # Other tests can leave a fake item map in the class level cache + TargetModel.item_map_cache = {} self.process = True orig_xread = redis.xread @@ -332,6 +360,16 @@ def test_overrides_complains_about_unknown_targets_packets_and_parameters(self): with self.assertRaisesRegex(RuntimeError, "does not exist"): override_tlm("INST", "HEALTH_STATUS", "BLAH", 1) + def test_overrides_and_normalizes_latest_in_every_packet_which_contains_the_item(self): + packet_names = TargetModel.get_item_to_packet_map("INST", scope="DEFAULT")["CCSDSSEQCNT"] + self.assertGreater(len(packet_names), 1) + override_tlm("INST LATEST CCSDSSEQCNT = 5", type="RAW") + self.assertCountEqual([o["packet_name"] for o in get_overrides()], packet_names) + for packet_name in packet_names: + self.assertEqual(tlm("INST", packet_name, "CCSDSSEQCNT", type="RAW"), 5) + normalize_tlm("INST LATEST CCSDSSEQCNT") + self.assertEqual(get_overrides(), []) + def test_overrides_complains_with_too_many_parameters(self): with self.assertRaisesRegex(RuntimeError, "Invalid number of arguments"): override_tlm("INST", "HEALTH_STATUS", "TEMP1", "TEMP2", 0.0) @@ -652,7 +690,7 @@ def test_complains_about_non_existant_packets(self): get_tlm_packet("INST BLAH") def test_complains_using_latest(self): - with self.assertRaisesRegex(RuntimeError, "Packet definition 'INST LATEST' does not exist"): + with self.assertRaisesRegex(RuntimeError, "LATEST is not supported by get_tlm_packet"): get_tlm_packet("INST", "LATEST") def test_complains_about_non_existant_value_types(self): @@ -835,6 +873,220 @@ def test_get_tlm_values_historical_passes_the_scope_to_the_tsdb_lookup(self): ) self.assertEqual(tsdb_lookup.call_args.kwargs["scope"], "OTHER") + def _latest_history_item(self, packet_name, item_name, value_type="CONVERTED"): + return TlmItem("INST", packet_name, item_name, value_type, None) + + @latest_candidates() + def test_get_tlm_values_historical_latest_queries_each_candidate_at_a_point_in_time(self): + item = self._latest_history_item + lookups = { + (item("HEALTH_STATUS", "TEMP2"),): [[[5.0, None]]], + (item("HEALTH_STATUS", "TEMP1"), item("HEALTH_STATUS", "PACKET_TIMESECONDS", "RAW")): [ + [[10.0, None], [100.0, None]] + ], + (item("PARAMS", "TEMP1"), item("PARAMS", "PACKET_TIMESECONDS", "RAW")): [[[20.0, "RED"], [200.0, None]]], + } + + def tsdb_lookup(query, start_time, scope, flatten): + self.assertFalse(flatten) + return lookups[tuple(query)] + + with patch.object(CvtModel, "tsdb_lookup", side_effect=tsdb_lookup): + vals = get_tlm_values( + ["INST__LATEST__TEMP1__CONVERTED", "INST__HEALTH_STATUS__TEMP2__CONVERTED"], + start_time="2026-09-13T00:00:00Z", + ) + self.assertEqual(vals, [[20.0, "RED"], [5.0, None]]) + + @latest_candidates() + def test_get_tlm_values_historical_latest_ignores_candidates_without_data(self): + def tsdb_lookup(query, **_kw): + return {} if query[0].packet_name == "PARAMS" else [[[10.0, None], [100.0, None]]] + + with patch.object(CvtModel, "tsdb_lookup", side_effect=tsdb_lookup): + vals = get_tlm_values(["INST__LATEST__TEMP1__CONVERTED"], start_time="2026-09-13T00:00:00Z") + self.assertEqual(vals, [[10.0, None]]) + + @latest_candidates() + def test_get_tlm_values_historical_latest_returns_nothing_when_nothing_is_found(self): + with patch.object(CvtModel, "tsdb_lookup", return_value={}): + vals = get_tlm_values( + ["INST__LATEST__TEMP1__CONVERTED", "INST__HEALTH_STATUS__TEMP2__CONVERTED"], + start_time="2026-09-13T00:00:00Z", + ) + self.assertEqual(vals, {}) + + @latest_candidates() + def test_get_tlm_values_historical_latest_joins_candidates_to_regular_rows_in_a_range(self): + item = self._latest_history_item + with patch.object(CvtModel, "tsdb_lookup") as tsdb_lookup: + tsdb_lookup.return_value = [ + [[5.0, None], [10.0, None], [300.0, None], [20.0, None], [200.0, None]], + [[6.0, None], [11.0, None], [300.0, None], [21.0, None], [400.0, None]], + [[7.0, None], [None, None], [None, None], [None, None], [None, None]], + ] + vals = get_tlm_values( + ["INST__LATEST__TEMP1__CONVERTED", "INST__HEALTH_STATUS__TEMP2__CONVERTED"], + start_time="2026-09-13T00:00:00Z", + end_time="2026-09-13T01:00:00Z", + ) + self.assertEqual( + tsdb_lookup.call_args[0][0], + [ + item("HEALTH_STATUS", "TEMP2"), + item("HEALTH_STATUS", "TEMP1"), + item("HEALTH_STATUS", "PACKET_TIMESECONDS", "RAW"), + item("PARAMS", "TEMP1"), + item("PARAMS", "PACKET_TIMESECONDS", "RAW"), + ], + ) + self.assertFalse(tsdb_lookup.call_args.kwargs["flatten"]) + self.assertEqual(vals, [[[10.0, None], [5.0, None]], [[21.0, None], [6.0, None]], [[None, None], [7.0, None]]]) + + @latest_candidates() + def test_get_tlm_values_historical_latest_range_of_only_latest_uses_the_current_packet(self): + with ( + patch.object(CvtModel, "determine_latest_packet_for_item", return_value="PARAMS"), + patch.object(CvtModel, "tsdb_lookup", return_value=[]) as tsdb_lookup, + ): + get_tlm_values( + ["INST__LATEST__TEMP1__CONVERTED"], + start_time="2026-09-13T00:00:00Z", + end_time="2026-09-13T01:00:00Z", + ) + self.assertEqual(tsdb_lookup.call_args[0][0], [self._latest_history_item("PARAMS", "TEMP1")]) + self.assertNotIn("flatten", tsdb_lookup.call_args.kwargs) + + @latest_candidates() + def test_get_tlm_values_historical_latest_skips_candidates_without_a_table_at_a_point_in_time(self): + item = self._latest_history_item + queries = [] + + def tsdb_lookup(query, **_kw): + queries.append(query) + return [[[10.0, None], [100.0, None]]] + + with ( + patch.object(CvtModel, "tsdb_packets_with_tables", return_value={("INST", "HEALTH_STATUS")}) as tables, + patch.object(CvtModel, "tsdb_lookup", side_effect=tsdb_lookup), + ): + vals = get_tlm_values(["INST__LATEST__TEMP1__CONVERTED"], start_time="2026-09-13T00:00:00Z") + tables.assert_called_once_with([("INST", "HEALTH_STATUS"), ("INST", "PARAMS")], scope="DEFAULT") + self.assertEqual( + queries, [[item("HEALTH_STATUS", "TEMP1"), item("HEALTH_STATUS", "PACKET_TIMESECONDS", "RAW")]] + ) + self.assertEqual(vals, [[10.0, None]]) + + @latest_candidates() + def test_get_tlm_values_historical_latest_skips_candidates_without_a_table_in_a_range(self): + item = self._latest_history_item + with ( + patch.object(CvtModel, "tsdb_packets_with_tables", return_value={("INST", "HEALTH_STATUS")}), + patch.object(CvtModel, "tsdb_lookup", return_value=[[[5.0, None], [10.0, None], [300.0, None]]]) as lookup, + ): + vals = get_tlm_values( + ["INST__LATEST__TEMP1__CONVERTED", "INST__HEALTH_STATUS__TEMP2__CONVERTED"], + start_time="2026-09-13T00:00:00Z", + end_time="2026-09-13T01:00:00Z", + ) + self.assertEqual( + lookup.call_args[0][0], + [ + item("HEALTH_STATUS", "TEMP2"), + item("HEALTH_STATUS", "TEMP1"), + item("HEALTH_STATUS", "PACKET_TIMESECONDS", "RAW"), + ], + ) + self.assertEqual(vals, [[10.0, None], [5.0, None]]) + + @latest_candidates() + def test_get_tlm_values_historical_latest_returns_none_when_no_candidate_has_a_table(self): + with ( + patch.object(CvtModel, "tsdb_packets_with_tables", return_value=set()), + patch.object(CvtModel, "tsdb_lookup", return_value=[[[5.0, None]]]) as lookup, + ): + vals = get_tlm_values( + ["INST__LATEST__TEMP1__CONVERTED", "INST__HEALTH_STATUS__TEMP2__CONVERTED"], + start_time="2026-09-13T00:00:00Z", + ) + lookup.assert_called_once() + self.assertEqual(vals, [[None, None], [5.0, None]]) + + @latest_candidates(params_temp1=PLAIN_TEMP1) + def test_get_tlm_values_historical_latest_queries_each_candidate_for_its_type_and_limits(self): + item = self._latest_history_item + queries = [] + + def tsdb_lookup(query, **_kw): + queries.append(query) + return {} + + with patch.object(CvtModel, "tsdb_lookup", side_effect=tsdb_lookup): + get_tlm_values(["INST__LATEST__TEMP1__FORMATTED__LIMITS"], start_time="2026-09-13T00:00:00Z") + self.assertCountEqual( + queries, + [ + [ + TlmItem("INST", "HEALTH_STATUS", "TEMP1", "FORMATTED", "LIMITS"), + item("HEALTH_STATUS", "PACKET_TIMESECONDS", "RAW"), + ], + [TlmItem("INST", "PARAMS", "TEMP1", "RAW", None), item("PARAMS", "PACKET_TIMESECONDS", "RAW")], + ], + ) + + @latest_candidates(params_temp1=PLAIN_TEMP1) + def test_get_tlm_values_historical_latest_range_of_only_latest_uses_the_current_packet_type_and_limits(self): + with ( + patch.object(CvtModel, "determine_latest_packet_for_item", return_value="PARAMS"), + patch.object(CvtModel, "tsdb_lookup", return_value=[]) as tsdb_lookup, + ): + get_tlm_values( + ["INST__LATEST__TEMP1__FORMATTED__LIMITS"], + start_time="2026-09-13T00:00:00Z", + end_time="2026-09-13T01:00:00Z", + ) + self.assertEqual(tsdb_lookup.call_args[0][0], [TlmItem("INST", "PARAMS", "TEMP1", "RAW", None)]) + + @latest_candidates(params_temp1=PLAIN_TEMP1) + def test_get_tlm_available_returns_the_best_type_and_limits_of_any_latest_packet(self): + with patch.object(CvtModel, "determine_latest_packet_for_item", return_value="PARAMS"): + self.assertEqual( + get_tlm_available(["INST__LATEST__TEMP1__FORMATTED"]), ["INST__LATEST__TEMP1__FORMATTED__LIMITS"] + ) + + @latest_candidates() + @patch("openc3.api.tlm_api.authorize") + def test_get_tlm_available_latest_authorizes_every_candidate(self, mock_authorize): + with patch.object(CvtModel, "determine_latest_packet_for_item", return_value="PARAMS"): + get_tlm_available(["INST__LATEST__TEMP1__CONVERTED"]) + self.assertCountEqual( + [call.kwargs["packet_name"] for call in mock_authorize.call_args_list], ["HEALTH_STATUS", "PARAMS"] + ) + + @latest_candidates() + @patch("openc3.api.tlm_api.authorize") + def test_get_tlm_values_historical_latest_authorizes_every_candidate(self, mock_authorize): + with patch.object(CvtModel, "tsdb_lookup", return_value={}): + get_tlm_values(["INST__LATEST__TEMP1__CONVERTED"], start_time="2026-09-13T00:00:00Z") + self.assertCountEqual( + [call.kwargs["packet_name"] for call in mock_authorize.call_args_list], ["HEALTH_STATUS", "PARAMS"] + ) + + def test_packet_apis_complain_that_latest_does_not_identify_a_packet(self): + for method in [get_tlm_buffer, get_tlm_packet, get_tlm, get_tlm_cnt, get_packet_derived_items]: + with self.assertRaisesRegex( + RuntimeError, f"LATEST is not supported by {method.__name__}\\(\\), specify a packet name" + ): + method("INST", "LATEST") + with self.assertRaisesRegex(RuntimeError, "LATEST is not supported"): + method("inst latest") + + def test_subscribe_packets_and_inject_tlm_complain_about_latest(self): + with self.assertRaisesRegex(RuntimeError, r"LATEST is not supported by subscribe_packets\(\)"): + subscribe_packets([["INST", "latest"]]) + with self.assertRaisesRegex(RuntimeError, r"LATEST is not supported by inject_tlm\(\)"): + inject_tlm("INST", "latest", {"TEMP1": 10}) + def test_get_tlm_values_accepts_the_limits_suffix_from_the_cvt(self): vals = get_tlm_values(["INST__HEALTH_STATUS__TEMP1__CONVERTED__LIMITS"]) self.assertEqual(vals[0][0], (-100.0)) @@ -1023,6 +1275,16 @@ def test_get_tlm_available_returns_proper_value_types_based_on_item_configuratio # DURATION should return RAW (no conversions) self.assertEqual(result[3], "INST__HEALTH_STATUS__DURATION__RAW") + def test_get_tlm_available_returns_raw_for_derived_items_with_a_read_conversion(self): + # Packet.decom stores a DERIVED item's read conversion as the raw value with no __C + items = [ + "INST__HEALTH_STATUS__DERIVED_GENERIC__CONVERTED", + "INST__HEALTH_STATUS__DERIVED_GENERIC__FORMATTED", + ] + result = get_tlm_available(items) + self.assertEqual(result[0], "INST__HEALTH_STATUS__DERIVED_GENERIC__RAW") + self.assertEqual(result[1], "INST__HEALTH_STATUS__DERIVED_GENERIC__RAW") + def test_get_tlm_available_handles_fallback_value_types(self): # Test fallback logic: WITH_UNITS -> FORMATTED -> CONVERTED -> RAW items = [ diff --git a/openc3/python/test/interfaces/protocols/test_template_protocol.py b/openc3/python/test/interfaces/protocols/test_template_protocol.py index 86e269b9dc..630ed4da4d 100644 --- a/openc3/python/test/interfaces/protocols/test_template_protocol.py +++ b/openc3/python/test/interfaces/protocols/test_template_protocol.py @@ -164,14 +164,15 @@ def test_waits_before_writing_during_the_initial_delay_period(self): self.interface.stream = TestTemplateProtocol.TemplateStream() self.interface.add_protocol(TemplateProtocol, ["0xABCD", "0xABCD", 0, 0.02], "READ_WRITE") packet = self._command_packet("SOUR'VOLT' , (self.)", VOLTAGE=1, CHANNEL=2) + # The initial delay is measured from connect, so start timing before it + start = time.time() self.interface.connect() - write = time.time() self.interface.write(packet) # The write really blocks until the initial delay has elapsed. Only the # lower bound can be checked here because a loaded machine can take # arbitrarily longer; test_waits_no_longer_than_the_initial_delay_before_writing # covers the protocol not waiting longer than it was configured to - self.assertGreaterEqual(time.time() - write, 0.02) + self.assertGreaterEqual(time.time() - start, 0.02) self.assertEqual(TestTemplateProtocol.write_buffer, b"SOUR'VOLT' 1, (self.2)\xab\xcd") def test_works_without_a_response(self): diff --git a/openc3/python/test/models/test_cvt_model.py b/openc3/python/test/models/test_cvt_model.py index 8348817e73..67849c9e05 100644 --- a/openc3/python/test/models/test_cvt_model.py +++ b/openc3/python/test/models/test_cvt_model.py @@ -609,6 +609,8 @@ def test_determine_latest_packet_for_item_works_if_not_received(self): model.create() names = TargetModel.names(scope="DEFAULT") self.assertIn("INST", names) + # Clear the item_map cache to avoid interference from previous tests + TargetModel.item_map_cache.pop("DEFAULT__INST__item_to_packet_map", None) Store.set("DEFAULT__INST__item_to_packet_map", json.dumps({"PACKET_ID": ["PACKET1"]})) Store.hset("DEFAULT__tlm__INST", "PACKET1", json.dumps({"PACKET_TIMESECONDS": None})) packet_name = CvtModel.determine_latest_packet_for_item("INST", "PACKET_ID", scope="DEFAULT") diff --git a/openc3/python/test/models/test_target_model.py b/openc3/python/test/models/test_target_model.py index 5169cc819c..21930ad0ab 100644 --- a/openc3/python/test/models/test_target_model.py +++ b/openc3/python/test/models/test_target_model.py @@ -12,6 +12,7 @@ # A portion of this file was funded by Blue Origin Enterprises, L.P. # See https://github.com/OpenC3/cosmos/pull/1953 +import json import unittest import unittest.mock from unittest.mock import patch @@ -143,7 +144,21 @@ def test_gets_item_to_packet_map_from_cache(self): cache = dict(TargetModel.item_map_cache) TargetModel.get_item_to_packet_map("INST", scope="DEFAULT") # Verify the cache time doesn't change - self.assertEqual(cache["INST"][0], (cache["INST"][0])) + self.assertEqual( + cache["DEFAULT__INST__item_to_packet_map"][0], + TargetModel.item_map_cache["DEFAULT__INST__item_to_packet_map"][0], + ) + + def test_shares_the_item_to_packet_map_cache_entry_for_a_lowercase_target(self): + TargetModel.get_item_to_packet_map("inst", scope="DEFAULT") + self.assertIn("DEFAULT__INST__item_to_packet_map", TargetModel.item_map_cache) + self.assertNotIn("DEFAULT__inst__item_to_packet_map", TargetModel.item_map_cache) + + def test_caches_the_item_to_packet_map_per_scope(self): + TargetModel.get_item_to_packet_map("INST", scope="DEFAULT") + Store.set("OTHER__INST__item_to_packet_map", json.dumps({"OTHER_ITEM": ["OTHER_PKT"]})) + self.assertEqual(TargetModel.get_item_to_packet_map("INST", scope="OTHER"), {"OTHER_ITEM": ["OTHER_PKT"]}) + self.assertIn("CCSDSVER", TargetModel.get_item_to_packet_map("INST", scope="DEFAULT")) def test_gets_item_to_packet_map_on_an_invalid_cache(self): TargetModel.get_item_to_packet_map("INST", scope="DEFAULT") @@ -153,7 +168,10 @@ def test_gets_item_to_packet_map_on_an_invalid_cache(self): TargetModel.ITEM_MAP_CACHE_TIMEOUT = 0 TargetModel.get_item_to_packet_map("INST", scope="DEFAULT") # Verify cached time was updated - self.assertLess(cache["INST"][0], TargetModel.item_map_cache["INST"][0]) + self.assertLess( + cache["DEFAULT__INST__item_to_packet_map"][0], + TargetModel.item_map_cache["DEFAULT__INST__item_to_packet_map"][0], + ) TargetModel.ITEM_MAP_CACHE_TIMEOUT = timeout def test_raises_for_an_unknown_type(self): diff --git a/openc3/python/test/script/test_api_shared.py b/openc3/python/test/script/test_api_shared.py index 45ed548989..81a95ae8fd 100644 --- a/openc3/python/test/script/test_api_shared.py +++ b/openc3/python/test/script/test_api_shared.py @@ -1040,6 +1040,10 @@ def test_waits_and_checks_a_logical_expression(self): ): wait_check_expression("'STRING' == STRING", 0.01) + def test_wait_packet_complains_about_latest(self): + with self.assertRaisesRegex(RuntimeError, "LATEST is not supported by wait_packet"): + wait_packet("INST", "latest", 1, 0.1) + def test_wait_packet_prints_warning_if_packet_not_received(self): global count count = False diff --git a/openc3/python/test/utilities/test_questdb_client.py b/openc3/python/test/utilities/test_questdb_client.py index 31427cf335..a51ac8332f 100644 --- a/openc3/python/test/utilities/test_questdb_client.py +++ b/openc3/python/test/utilities/test_questdb_client.py @@ -34,6 +34,21 @@ def test_returns_a_row_of_nones_for_a_placeholder_without_an_end_time(self): [[None, None]], ) + def test_returns_a_list_of_rows_for_placeholders_when_not_flattening(self): + self.assertEqual( + QuestDBClient.tsdb_lookup([[None] * 5], start_time="2026-09-13T00:00:00Z", flatten=False), + [[[None, None]]], + ) + + def test_passes_flatten_to_a_single_db_shard_lookup(self): + with patch.object(QuestDBClient, "_tsdb_lookup_single_db_shard", return_value=[[[1.0, None]]]) as single: + QuestDBClient.tsdb_lookup( + [["INST", "HEALTH_STATUS", "TEMP1", "CONVERTED", None]], + start_time="2026-09-13T00:00:00Z", + flatten=False, + ) + self.assertFalse(single.call_args.kwargs["flatten"]) + class TestTsdbLookupDbShards(unittest.TestCase): ITEMS = [ @@ -78,6 +93,10 @@ def test_queries_each_db_shard_and_merges_a_single_row(self): for call in single.call_args_list: self.assertFalse(call.kwargs["flatten"]) + def test_keeps_a_single_merged_row_as_a_list_of_rows_when_not_flattening(self): + result, _ = self.lookup({0: [[[1.0, None], [3.0, None]]], 1: [[[2.0, None]]]}, flatten=False) + self.assertEqual(result, [[[1.0, None], [2.0, None], [3.0, None]]]) + def test_merges_multiple_rows_filling_missing_rows_with_nones(self): result, _ = self.lookup( {0: [[[1.0, None], [3.0, None]], [[4.0, None], [6.0, None]]], 1: [[[2.0, None]]]}, @@ -124,6 +143,32 @@ def test_queries_and_reconnects_the_given_db_shard(self): disconnect.assert_called_once_with(db_shard=2) +class TestTlmPacketsWithTables(unittest.TestCase): + def test_returns_the_packets_with_a_table_queried_by_db_shard(self): + tables = {0: {"DEFAULT__TLM__INST__HEALTH_STATUS"}, 1: {"DEFAULT__TLM__INST2__ADCS"}} + packets = [("INST", "HEALTH_STATUS"), ("INST", "PARAMS"), ("INST2", "ADCS")] + with ( + patch.object(QuestDBClient, "db_shard_for_target", side_effect=lambda t, scope: 1 if t == "INST2" else 0), + patch.object(QuestDBClient, "table_names", side_effect=lambda db_shard: tables[db_shard]) as table_names, + ): + result = QuestDBClient.tlm_packets_with_tables(packets) + self.assertEqual(result, {("INST", "HEALTH_STATUS"), ("INST2", "ADCS")}) + self.assertEqual(table_names.call_count, 2) + + def test_assumes_all_the_packets_have_a_table_if_the_table_names_cannot_be_fetched(self): + packets = [("INST", "HEALTH_STATUS"), ("INST", "PARAMS")] + with ( + patch.object(QuestDBClient, "db_shard_for_target", return_value=0), + patch.object(QuestDBClient, "table_names", return_value=None), + ): + self.assertEqual(QuestDBClient.tlm_packets_with_tables(packets), set(packets)) + + def test_does_not_query_without_packets(self): + with patch.object(QuestDBClient, "table_names") as table_names: + self.assertEqual(QuestDBClient.tlm_packets_with_tables([]), set()) + table_names.assert_not_called() + + class TestTableNames(unittest.TestCase): def test_returns_the_upcased_table_names(self): rows = [{"table_name": "DEFAULT__TLM__INST__PARAMS"}, {"table_name": "default__cmd__inst__collect"}] diff --git a/openc3/spec/api/limits_api_spec.rb b/openc3/spec/api/limits_api_spec.rb index fb081cd8fd..01a50e88de 100644 --- a/openc3/spec/api/limits_api_spec.rb +++ b/openc3/spec/api/limits_api_spec.rb @@ -90,6 +90,21 @@ def with_decom_microservice eql({ 'DEFAULT' => [-80.0, -70.0, 60.0, 80.0, -20.0, 20.0], 'TVAC' => [-80.0, -30.0, 30.0, 80.0] }) end end + + it "resolves a LATEST item in the requested scope" do + model = TargetModel.new(folder_name: 'INST', name: 'INST', scope: "OTHER") + model.create + model.update_store(System.new(['INST'], File.join(SPEC_DIR, 'install', 'config', 'targets'))) + expect(CvtModel).to receive(:determine_latest_packet_for_item).with('INST', 'TEMP1', cache_timeout: nil, scope: 'OTHER').and_call_original + expect(@api.get_limits("INST", "LATEST", "TEMP1", scope: "OTHER")).to \ + eql({ 'DEFAULT' => [-80.0, -70.0, 60.0, 80.0, -20.0, 20.0], 'TVAC' => [-80.0, -30.0, 30.0, 80.0] }) + end + + it "authorizes the resolved packet rather than LATEST" do + allow(CvtModel).to receive(:determine_latest_packet_for_item).and_return('HEALTH_STATUS') + expect(@api).to receive(:authorize).with(hash_including(target_name: 'INST', packet_name: 'HEALTH_STATUS')) + @api.get_limits("inst", "latest", "temp1") + end end describe "set_limits" do @@ -218,6 +233,43 @@ def with_decom_microservice end end + describe "LATEST limits changes" do + it "enables and disables limits in every packet which contains the item" do + packet_names = TargetModel.get_item_to_packet_map('INST', scope: 'DEFAULT')['CCSDSSEQCNT'] + expect(packet_names.length).to be > 1 + @api.disable_limits("INST LATEST CCSDSSEQCNT") + packet_names.each do |packet_name| + expect(@api.limits_enabled?("INST", packet_name, "CCSDSSEQCNT")).to be false + end + @api.enable_limits("INST", "LATEST", "CCSDSSEQCNT") + packet_names.each do |packet_name| + expect(@api.limits_enabled?("INST", packet_name, "CCSDSSEQCNT")).to be true + end + end + + it "sets limits through LATEST" do + @api.set_limits("INST", "LATEST", "TEMP1", 0.0, 10.0, 20.0, 30.0) + expect(@api.get_limits("INST", "HEALTH_STATUS", "TEMP1")['CUSTOM']).to eql([0.0, 10.0, 20.0, 30.0]) + end + + it "sets limits in no packet if any packet fails" do + allow(TargetModel).to receive(:get_item_to_packet_map).and_return({ 'TEMP1' => ['HEALTH_STATUS', 'MECH'] }) + expect { @api.set_limits("INST", "LATEST", "TEMP1", 0.0, 10.0, 20.0, 30.0) }.to \ + raise_error(RuntimeError, "Item 'INST MECH TEMP1' does not exist while setting limits") + expect(@api.get_limits("INST", "HEALTH_STATUS", "TEMP1")['CUSTOM']).to be_nil + end + + it "sets the state color in every packet which contains the item" do + packet_names = TargetModel.get_item_to_packet_map('INST', scope: 'DEFAULT')['CCSDSTYPE'] + expect(packet_names.length).to be > 1 + @api.set_state_color("INST", "LATEST", "CCSDSTYPE", "TLM", "GREEN") + packet_names.each do |packet_name| + item = TargetModel.packet_item("INST", packet_name, "CCSDSTYPE", scope: "DEFAULT") + expect(item['states']['TLM']['color']).to eql('GREEN') + end + end + end + describe "get_limits_groups" do it "returns an empty hash with no groups" do # Remove all limits_groups diff --git a/openc3/spec/api/tlm_api_spec.rb b/openc3/spec/api/tlm_api_spec.rb index ab7db8f41f..debc8f114e 100644 --- a/openc3/spec/api/tlm_api_spec.rb +++ b/openc3/spec/api/tlm_api_spec.rb @@ -392,6 +392,20 @@ def test_tlm_unknown(method) end end + describe "override_tlm and normalize_tlm with LATEST" do + it "overrides and normalizes the item in every packet which contains it" do + packet_names = TargetModel.get_item_to_packet_map('INST', scope: 'DEFAULT')['CCSDSSEQCNT'] + expect(packet_names.length).to be > 1 + @api.override_tlm("INST LATEST CCSDSSEQCNT = 5", type: :RAW) + expect(@api.get_overrides().map { |o| o['packet_name'] }).to match_array(packet_names) + packet_names.each do |packet_name| + expect(@api.tlm("INST", packet_name, "CCSDSSEQCNT", type: :RAW)).to eql(5) + end + @api.normalize_tlm("INST LATEST CCSDSSEQCNT") + expect(@api.get_overrides()).to eql([]) + end + end + describe "get_overrides" do it "returns empty array with no overrides" do expect(@api.get_overrides()).to eql([]) @@ -545,6 +559,20 @@ def test_tlm_unknown(method) end end + describe "packet APIs with LATEST" do + it "complains that LATEST does not identify a packet" do + %w(get_tlm_buffer get_tlm_packet get_tlm get_tlm_cnt get_packet_derived_items).each do |method_name| + expect { @api.public_send(method_name, "INST", "LATEST") }.to raise_error(RuntimeError, "ERROR: LATEST is not supported by #{method_name}(), specify a packet name") + expect { @api.public_send(method_name, "inst latest") }.to raise_error(RuntimeError, /LATEST is not supported/) + end + end + + it "complains about LATEST in subscribe_packets and inject_tlm" do + expect { @api.subscribe_packets([["INST", "latest"]]) }.to raise_error(RuntimeError, "ERROR: LATEST is not supported by subscribe_packets(), specify a packet name") + expect { @api.inject_tlm("INST", "latest", { TEMP1: 10 }) }.to raise_error(RuntimeError, "ERROR: LATEST is not supported by inject_tlm(), specify a packet name") + end + end + describe "get_tlm_packet" do it "complains about non-existent targets" do expect { @api.get_tlm_packet("BLAH", "HEALTH_STATUS") }.to raise_error(RuntimeError, "Packet definition 'BLAH HEALTH_STATUS' does not exist") @@ -555,7 +583,7 @@ def test_tlm_unknown(method) end it "complains using LATEST" do - expect { @api.get_tlm_packet("INST", "LATEST") }.to raise_error(RuntimeError, "Packet definition 'INST LATEST' does not exist") + expect { @api.get_tlm_packet("INST", "LATEST") }.to raise_error(RuntimeError, "ERROR: LATEST is not supported by get_tlm_packet(), specify a packet name") end it "complains about non-existent value_types" do @@ -685,6 +713,31 @@ def test_tlm_unknown(method) end describe "get_tlm_available" do + it "returns the best value type and limits of any LATEST packet" do + allow(TargetModel).to receive(:get_item_to_packet_map).and_return({ 'TEMP1' => ['PARAMS', 'HEALTH_STATUS'] }) + allow(CvtModel).to receive(:determine_latest_packet_for_item).and_return('PARAMS') + allow(TargetModel).to receive(:packet_item).and_call_original + plain = { 'name' => 'TEMP1', 'data_type' => 'UINT', 'limits' => {} } + allow(TargetModel).to receive(:packet_item).with('INST', 'PARAMS', 'TEMP1', scope: 'DEFAULT').and_return(plain) + expect(@api.get_tlm_available(['INST__LATEST__TEMP1__FORMATTED'])).to eql(['INST__LATEST__TEMP1__FORMATTED__LIMITS']) + end + + it "authorizes every LATEST packet before reading its definition" do + allow(TargetModel).to receive(:get_item_to_packet_map).and_return({ 'TEMP1' => ['PARAMS', 'HEALTH_STATUS'] }) + allow(CvtModel).to receive(:determine_latest_packet_for_item).and_return('PARAMS') + allow(@api).to receive(:authorize) do |packet_name:, **| + raise ForbiddenError, 'forbidden' if packet_name == 'HEALTH_STATUS' + end + expect(TargetModel).not_to receive(:packet_item) + expect { @api.get_tlm_available(['INST__LATEST__TEMP1__FORMATTED']) }.to raise_error(ForbiddenError) + end + + it "returns RAW for DERIVED items with a read conversion" do + # Packet#decom stores a DERIVED item's read conversion as the raw value with no __C + items = ['INST__HEALTH_STATUS__TEMP1HIGH__CONVERTED', 'INST__HEALTH_STATUS__TEMP1HIGH__FORMATTED'] + expect(@api.get_tlm_available(items)).to eql(['INST__HEALTH_STATUS__TEMP1HIGH__RAW', 'INST__HEALTH_STATUS__TEMP1HIGH__RAW']) + end + it "returns a valid list of items based on inputs" do items = [] # Ask for WITH_UNITS for an item which has various formats @@ -783,6 +836,129 @@ def test_tlm_unknown(method) expect(vals).to eql([[0.0, nil], [-100.0, "RED_LOW"], [nil, nil]]) end + context "with a historical LATEST item" do + before(:each) do + allow(TargetModel).to receive(:get_item_to_packet_map).and_return({ 'TEMP1' => ['HEALTH_STATUS', 'PARAMS'] }) + # Pretend PARAMS has the same TEMP1 as HEALTH_STATUS + temp1 = TargetModel.packet_item('INST', 'HEALTH_STATUS', 'TEMP1', scope: 'DEFAULT') + allow(TargetModel).to receive(:packet_item).and_call_original + allow(TargetModel).to receive(:packet_item).with('INST', 'PARAMS', 'TEMP1', scope: 'DEFAULT').and_return(temp1) + allow(CvtModel).to receive(:tsdb_packets_with_tables) { |packets, **| packets.to_set } + end + + def item(packet_name, item_name, value_type = "CONVERTED") + ["INST", packet_name, item_name, value_type, nil] + end + + let(:items) { ["INST__LATEST__TEMP1__CONVERTED", "INST__HEALTH_STATUS__TEMP2__CONVERTED"] } + + it "queries each candidate packet at a point in time and returns the newest" do + lookups = { + [item("HEALTH_STATUS", "TEMP2")] => [[[5.0, nil]]], + [item("HEALTH_STATUS", "TEMP1"), item("HEALTH_STATUS", "PACKET_TIMESECONDS", "RAW")] => [[[10.0, nil], [100.0, nil]]], + [item("PARAMS", "TEMP1"), item("PARAMS", "PACKET_TIMESECONDS", "RAW")] => [[[20.0, "RED"], [200.0, nil]]], + } + allow(CvtModel).to receive(:tsdb_lookup) do |query, start_time:, scope:, flatten:| + expect(start_time).to eql("2026-09-13T00:00:00Z") + expect(flatten).to be false + lookups.fetch(query) + end + expect(@api.get_tlm_values(items, start_time: "2026-09-13T00:00:00Z")).to eql([[20.0, "RED"], [5.0, nil]]) + end + + it "ignores candidate packets without data at a point in time" do + allow(CvtModel).to receive(:tsdb_lookup) do |query, **| + query[0][1] == "PARAMS" ? {} : [[[10.0, nil], [100.0, nil]]] + end + expect(@api.get_tlm_values(["INST__LATEST__TEMP1__CONVERTED"], start_time: "2026-09-13T00:00:00Z")).to eql([[10.0, nil]]) + end + + it "returns no results when nothing is found at a point in time" do + allow(CvtModel).to receive(:tsdb_lookup).and_return({}) + expect(@api.get_tlm_values(items, start_time: "2026-09-13T00:00:00Z")).to eql({}) + end + + it "joins the candidates to the rows of the regular items in a range" do + expect(CvtModel).to receive(:tsdb_lookup).with( + [ + item("HEALTH_STATUS", "TEMP2"), + item("HEALTH_STATUS", "TEMP1"), + item("HEALTH_STATUS", "PACKET_TIMESECONDS", "RAW"), + item("PARAMS", "TEMP1"), + item("PARAMS", "PACKET_TIMESECONDS", "RAW"), + ], + start_time: "2026-09-13T00:00:00Z", end_time: "2026-09-13T01:00:00Z", scope: "DEFAULT", flatten: false + ).and_return([ + [[5.0, nil], [10.0, nil], [300.0, nil], [20.0, nil], [200.0, nil]], + [[6.0, nil], [11.0, nil], [300.0, nil], [21.0, nil], [400.0, nil]], + [[7.0, nil], [nil, nil], [nil, nil], [nil, nil], [nil, nil]], + ]) + vals = @api.get_tlm_values(items, start_time: "2026-09-13T00:00:00Z", end_time: "2026-09-13T01:00:00Z") + expect(vals).to eql([[[10.0, nil], [5.0, nil]], [[21.0, nil], [6.0, nil]], [[nil, nil], [7.0, nil]]]) + end + + it "uses the packet received most recently now for a range of only LATEST items" do + allow(CvtModel).to receive(:determine_latest_packet_for_item).and_return("PARAMS") + expect(CvtModel).to receive(:tsdb_lookup).with( + [item("PARAMS", "TEMP1")], start_time: "2026-09-13T00:00:00Z", end_time: "2026-09-13T01:00:00Z", scope: "DEFAULT" + ).and_return([[[[20.0, nil]]], [[[21.0, nil]]]]) + @api.get_tlm_values(["INST__LATEST__TEMP1__CONVERTED"], start_time: "2026-09-13T00:00:00Z", end_time: "2026-09-13T01:00:00Z") + end + + it "queries each candidate for the value type and limits it has" do + plain = { 'name' => 'TEMP1', 'data_type' => 'UINT', 'limits' => {} } + allow(TargetModel).to receive(:packet_item).with('INST', 'PARAMS', 'TEMP1', scope: 'DEFAULT').and_return(plain) + queries = [] + allow(CvtModel).to receive(:tsdb_lookup) { |query, **| queries << query; {} } + @api.get_tlm_values(["INST__LATEST__TEMP1__FORMATTED__LIMITS"], start_time: "2026-09-13T00:00:00Z") + expect(queries).to contain_exactly( + [["INST", "HEALTH_STATUS", "TEMP1", "FORMATTED", "LIMITS"], item("HEALTH_STATUS", "PACKET_TIMESECONDS", "RAW")], + [["INST", "PARAMS", "TEMP1", "RAW", nil], item("PARAMS", "PACKET_TIMESECONDS", "RAW")], + ) + end + + it "queries the current packet for the value type and limits it has in a range of only LATEST items" do + plain = { 'name' => 'TEMP1', 'data_type' => 'UINT', 'limits' => {} } + allow(TargetModel).to receive(:packet_item).with('INST', 'PARAMS', 'TEMP1', scope: 'DEFAULT').and_return(plain) + allow(CvtModel).to receive(:determine_latest_packet_for_item).and_return("PARAMS") + expect(CvtModel).to receive(:tsdb_lookup).with( + [["INST", "PARAMS", "TEMP1", "RAW", nil]], start_time: "2026-09-13T00:00:00Z", end_time: "2026-09-13T01:00:00Z", scope: "DEFAULT" + ).and_return([]) + @api.get_tlm_values(["INST__LATEST__TEMP1__FORMATTED__LIMITS"], start_time: "2026-09-13T00:00:00Z", end_time: "2026-09-13T01:00:00Z") + end + + it "skips candidate packets without a table at a point in time" do + expect(CvtModel).to receive(:tsdb_packets_with_tables).with([["INST", "HEALTH_STATUS"], ["INST", "PARAMS"]], scope: "DEFAULT").and_return(Set[["INST", "HEALTH_STATUS"]]) + queries = [] + allow(CvtModel).to receive(:tsdb_lookup) { |query, **| queries << query; [[[10.0, nil], [100.0, nil]]] } + expect(@api.get_tlm_values(["INST__LATEST__TEMP1__CONVERTED"], start_time: "2026-09-13T00:00:00Z")).to eql([[10.0, nil]]) + expect(queries).to eql([[item("HEALTH_STATUS", "TEMP1"), item("HEALTH_STATUS", "PACKET_TIMESECONDS", "RAW")]]) + end + + it "skips candidate packets without a table in a range" do + allow(CvtModel).to receive(:tsdb_packets_with_tables).and_return(Set[["INST", "HEALTH_STATUS"]]) + expect(CvtModel).to receive(:tsdb_lookup).with( + [item("HEALTH_STATUS", "TEMP2"), item("HEALTH_STATUS", "TEMP1"), item("HEALTH_STATUS", "PACKET_TIMESECONDS", "RAW")], + start_time: "2026-09-13T00:00:00Z", end_time: "2026-09-13T01:00:00Z", scope: "DEFAULT", flatten: false + ).and_return([[[5.0, nil], [10.0, nil], [300.0, nil]]]) + vals = @api.get_tlm_values(items, start_time: "2026-09-13T00:00:00Z", end_time: "2026-09-13T01:00:00Z") + expect(vals).to eql([[10.0, nil], [5.0, nil]]) + end + + it "returns nil for a LATEST item when no candidate has a table" do + allow(CvtModel).to receive(:tsdb_packets_with_tables).and_return(Set.new) + expect(CvtModel).to receive(:tsdb_lookup).once.with([item("HEALTH_STATUS", "TEMP2")], any_args).and_return([[[5.0, nil]]]) + expect(@api.get_tlm_values(items, start_time: "2026-09-13T00:00:00Z")).to eql([[nil, nil], [5.0, nil]]) + end + + it "authorizes every candidate packet" do + allow(CvtModel).to receive(:tsdb_lookup).and_return({}) + expect(@api).to receive(:authorize).with(hash_including(packet_name: "HEALTH_STATUS")) + expect(@api).to receive(:authorize).with(hash_including(packet_name: "PARAMS")) + @api.get_tlm_values(["INST__LATEST__TEMP1__CONVERTED"], start_time: "2026-09-13T00:00:00Z") + end + end + it "accepts the limits suffix from get_tlm_available" do vals = @api.get_tlm_values(["INST__HEALTH_STATUS__TEMP1__CONVERTED__LIMITS"]) expect(vals[0][0]).to eql(-100.0) diff --git a/openc3/spec/install/api_docs_spec.rb b/openc3/spec/install/api_docs_spec.rb index 670cea8167..4b9dede2b6 100644 --- a/openc3/spec/install/api_docs_spec.rb +++ b/openc3/spec/install/api_docs_spec.rb @@ -59,14 +59,14 @@ def parse_file(filename, methods) def ruby_api ruby_api = {} Dir[File.join(SPEC_DIR,'../lib/openc3/script/*.rb')].each do |filename| - next if filename.include?('extract') - next if filename.include?('web_socket_api') - next if filename.include?('suite_results') - next if filename.include?('suite_runner') + next if File.basename(filename).include?('extract') + next if File.basename(filename).include?('web_socket_api') + next if File.basename(filename).include?('suite_results') + next if File.basename(filename).include?('suite_runner') parse_file(filename, ruby_api) end Dir[File.join(SPEC_DIR,'../lib/openc3/api/*.rb')].each do |filename| - next if filename.include?('metrics_api') # TODO: document and implement Python equivalent + next if File.basename(filename).include?('metrics_api') # TODO: document and implement Python equivalent parse_file(filename, ruby_api) end return ruby_api @@ -75,13 +75,13 @@ def ruby_api def python_api python_api = {} Dir[File.join(SPEC_DIR,'../python/openc3/script/*.py')].each do |filename| - next if filename.include?('authorization') - next if filename.include?('decorators') - next if filename.include?('server_proxy') - next if filename.include?('stream') - next if filename.include?('web_socket_api') - next if filename.include?('suite_results') - next if filename.include?('suite_runner') + next if File.basename(filename).include?('authorization') + next if File.basename(filename).include?('decorators') + next if File.basename(filename).include?('server_proxy') + next if File.basename(filename).include?('stream') + next if File.basename(filename).include?('web_socket_api') + next if File.basename(filename).include?('suite_results') + next if File.basename(filename).include?('suite_runner') parse_file(filename, python_api) end Dir[File.join(SPEC_DIR,'../python/openc3/api/*.py')].each do |filename| diff --git a/openc3/spec/models/cvt_model_spec.rb b/openc3/spec/models/cvt_model_spec.rb index 00f62536a5..9719487de6 100644 --- a/openc3/spec/models/cvt_model_spec.rb +++ b/openc3/spec/models/cvt_model_spec.rb @@ -357,7 +357,7 @@ def check_temp1 names = TargetModel.names(scope:"DEFAULT") expect(names.include?("INST")) # Clear the item_map cache to avoid interference from previous tests - TargetModel.class_variable_get(:@@item_map_cache)["INST"] = nil + TargetModel.class_variable_get(:@@item_map_cache)["DEFAULT__INST__item_to_packet_map"] = nil Store.set("DEFAULT__INST__item_to_packet_map", JSON.generate({"PACKET_ID" => ["PACKET1"]})) Store.hset("DEFAULT__tlm__INST", "PACKET1", JSON.generate({"PACKET_TIMESECONDS" => nil})) packet_name = CvtModel.determine_latest_packet_for_item("INST", "PACKET_ID", scope:"DEFAULT") diff --git a/openc3/spec/models/target_model_spec.rb b/openc3/spec/models/target_model_spec.rb index acd35ab7f3..3aa3c47cbc 100644 --- a/openc3/spec/models/target_model_spec.rb +++ b/openc3/spec/models/target_model_spec.rb @@ -462,7 +462,21 @@ def shadow_response(body) expect(itpm["CCSDSVER"]).to eql(%w(ADCS HEALTH_STATUS HIDDEN IMAGE MECH PARAMS)) # Verify cached time was NOT updated cache = TargetModel.class_variable_get(:@@item_map_cache) - expect(cache["INST"][0]).to eq(orig_cache["INST"][0]) + expect(cache["DEFAULT__INST__item_to_packet_map"][0]).to eq(orig_cache["DEFAULT__INST__item_to_packet_map"][0]) + end + + it "shares the item-to-packet map cache entry for a lowercase target" do + TargetModel.get_item_to_packet_map("inst", scope: "DEFAULT") + cache = TargetModel.class_variable_get(:@@item_map_cache) + expect(cache.keys).to include("DEFAULT__INST__item_to_packet_map") + expect(cache.keys).not_to include("DEFAULT__inst__item_to_packet_map") + end + + it "caches the item-to-packet map per scope" do + TargetModel.get_item_to_packet_map("INST", scope: "DEFAULT") + Store.set("OTHER__INST__item_to_packet_map", JSON.generate({ "OTHER_ITEM" => ["OTHER_PKT"] })) + expect(TargetModel.get_item_to_packet_map("INST", scope: "OTHER")).to eql({ "OTHER_ITEM" => ["OTHER_PKT"] }) + expect(TargetModel.get_item_to_packet_map("INST", scope: "DEFAULT")["CCSDSVER"]).to be_a(Array) end it "gets item-to-packet map on an invalid cache" do @@ -477,7 +491,7 @@ def shadow_response(body) expect(itpm["CCSDSVER"]).to eql(%w(ADCS HEALTH_STATUS HIDDEN IMAGE MECH PARAMS)) # Verify cached time was updated cache = TargetModel.class_variable_get(:@@item_map_cache) - expect(cache["INST"][0]).to be > orig_cache["INST"][0] + expect(cache["DEFAULT__INST__item_to_packet_map"][0]).to be > orig_cache["DEFAULT__INST__item_to_packet_map"][0] OpenC3.disable_warnings do TargetModel::ITEM_MAP_CACHE_TIMEOUT = timeout end diff --git a/openc3/spec/script/api_shared_spec.rb b/openc3/spec/script/api_shared_spec.rb index 1f974362b7..2719ab3234 100644 --- a/openc3/spec/script/api_shared_spec.rb +++ b/openc3/spec/script/api_shared_spec.rb @@ -860,6 +860,10 @@ def openc3_script_sleep(_sleep_time = nil) @sleep_cancel = cancel end + it "complains about LATEST" do + expect { wait_packet("INST", "latest", 1, 0.1) }.to raise_error(/LATEST is not supported by wait_packet/) + end + it "prints warning if packet not received" do @count = false capture_io do |stdout| diff --git a/openc3/spec/utilities/questdb_client_spec.rb b/openc3/spec/utilities/questdb_client_spec.rb index 3e9081aafb..214c4921c1 100644 --- a/openc3/spec/utilities/questdb_client_spec.rb +++ b/openc3/spec/utilities/questdb_client_spec.rb @@ -29,6 +29,16 @@ module OpenC3 expect(QuestDBClient.tsdb_lookup([Array.new(5)], start_time: "2026-09-13T00:00:00Z")).to eq([[nil, nil]]) end + it "returns a list of rows for placeholders when not flattening" do + expect(QuestDBClient.tsdb_lookup([Array.new(5)], start_time: "2026-09-13T00:00:00Z", flatten: false)).to eq([[[nil, nil]]]) + end + + it "passes flatten to a single db_shard lookup" do + allow(QuestDBClient).to receive(:tsdb_lookup_single_db_shard).and_return([[[1.0, nil]]]) + QuestDBClient.tsdb_lookup([["INST", "HEALTH_STATUS", "TEMP1", "CONVERTED", nil]], start_time: "2026-09-13T00:00:00Z", flatten: false) + expect(QuestDBClient).to have_received(:tsdb_lookup_single_db_shard).with(anything, hash_including(flatten: false)) + end + context "with items across db_shards" do let(:items) do [ @@ -38,14 +48,14 @@ module OpenC3 ] end - def lookup(results, end_time: nil) + def lookup(results, end_time: nil, flatten: true) # INST is on db_shard 0 and INST2 is on db_shard 1 allow(QuestDBClient).to receive(:db_shard_for_target) { |target_name, **| target_name == "INST2" ? 1 : 0 } allow(QuestDBClient).to receive(:tsdb_lookup_single_db_shard) do |_items, db_shard:, flatten:, **| expect(flatten).to be false results[db_shard] end - QuestDBClient.tsdb_lookup(items, start_time: "2026-09-13T00:00:00Z", end_time: end_time) + QuestDBClient.tsdb_lookup(items, start_time: "2026-09-13T00:00:00Z", end_time: end_time, flatten: flatten) end it "queries each db_shard and merges a single row" do @@ -55,6 +65,11 @@ def lookup(results, end_time: nil) expect(QuestDBClient).to have_received(:tsdb_lookup_single_db_shard).with([items[1]], hash_including(db_shard: 1)) end + it "keeps a single merged row as a list of rows when not flattening" do + result = lookup({ 0 => [[[1.0, nil], [3.0, nil]]], 1 => [[[2.0, nil]]] }, flatten: false) + expect(result).to eq([[[1.0, nil], [2.0, nil], [3.0, nil]]]) + end + it "merges a single row of a multi-row query" do # A db_shard with a single row must not be mistaken for multiple rows result = lookup({ 0 => [[[1.0, nil], [3.0, nil]]], 1 => [[[2.0, nil]]] }, end_time: "2026-09-13T01:00:00Z") @@ -76,6 +91,28 @@ def lookup(results, end_time: nil) end end + describe "tlm_packets_with_tables" do + it "returns the packets with a table queried by db_shard" do + allow(QuestDBClient).to receive(:db_shard_for_target) { |target_name, **| target_name == "INST2" ? 1 : 0 } + tables = { 0 => Set["DEFAULT__TLM__INST__HEALTH_STATUS"], 1 => Set["DEFAULT__TLM__INST2__ADCS"] } + expect(QuestDBClient).to receive(:table_names).twice { |db_shard:| tables[db_shard] } + packets = [["INST", "HEALTH_STATUS"], ["INST", "PARAMS"], ["INST2", "ADCS"]] + expect(QuestDBClient.tlm_packets_with_tables(packets)).to eql(Set[["INST", "HEALTH_STATUS"], ["INST2", "ADCS"]]) + end + + it "assumes all the packets have a table if the table names can't be fetched" do + allow(QuestDBClient).to receive(:db_shard_for_target).and_return(0) + allow(QuestDBClient).to receive(:table_names).and_return(nil) + packets = [["INST", "HEALTH_STATUS"], ["INST", "PARAMS"]] + expect(QuestDBClient.tlm_packets_with_tables(packets)).to eql(Set.new(packets)) + end + + it "doesn't query without packets" do + expect(QuestDBClient).not_to receive(:table_names) + expect(QuestDBClient.tlm_packets_with_tables([])).to eql(Set.new) + end + end + describe "table_names" do it "returns the upcased table names" do result = double("PG::Result", values: [["DEFAULT__TLM__INST__PARAMS"], ["default__cmd__inst__collect"]]) diff --git a/openc3/test/integration/tsdb/python/test_tlm_api_latest_tsdb.py b/openc3/test/integration/tsdb/python/test_tlm_api_latest_tsdb.py new file mode 100644 index 0000000000..a9e08a2ce8 --- /dev/null +++ b/openc3/test/integration/tsdb/python/test_tlm_api_latest_tsdb.py @@ -0,0 +1,139 @@ +# Copyright 2026 OpenC3, Inc. +# All Rights Reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE.md for more details. +# +# This file may also be used under the terms of a commercial license +# if purchased from OpenC3, Inc. + +""" +End-to-end tests for historical get_tlm_values with LATEST items. + +LATEST must return the value from whichever packet containing the item was +received most recently at the requested time, not the packet received most +recently now, and not just the first packet in the QuestDB ASOF JOIN. + +Run with: + 1. Start QuestDB: docker compose -f docker-compose.test.yml up -d + 2. Run tests: uv run pytest ../test/integration/tsdb/python/test_tlm_api_latest_tsdb.py (from openc3/python) + 3. Stop QuestDB: docker compose -f docker-compose.test.yml down +""" + +import os +import sys +import time +from datetime import datetime, timezone +from unittest.mock import patch + +import pytest + + +# Add openc3 python path for imports +# Path: openc3/test/integration/tsdb/python -> openc3/python +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "python")) + +os.environ.setdefault("OPENC3_TSDB_HOSTNAME", "127.0.0.1") +os.environ.setdefault("OPENC3_TSDB_INGEST_PORT", "9000") +os.environ.setdefault("OPENC3_TSDB_QUERY_PORT", "8812") +os.environ.setdefault("OPENC3_TSDB_USERNAME", "admin") +os.environ.setdefault("OPENC3_TSDB_PASSWORD", "admin") +os.environ.setdefault("OPENC3_SCOPE", "DEFAULT") + +from conftest import requires_questdb + +from openc3.api.tlm_api import get_tlm_values +from openc3.models.target_model import TargetModel +from openc3.utilities.questdb_client import QuestDBClient + + +TARGET = "PY_LT" +PACKET_DEF = {"items": [{"name": "VALUE", "data_type": "INT", "bit_size": 32}]} +# Only PKT_B has units so only it has a formatted (VALUE__F) column +UNITS_PACKET_DEF = {"items": [{"name": "VALUE", "data_type": "INT", "bit_size": 32, "units": "degC"}]} +PACKET_DEFS = {"PKT_A": PACKET_DEF, "PKT_B": UNITS_PACKET_DEF, "REGULAR": PACKET_DEF} +# Timeline in ms from the base time (VALUE written to each packet) +ROWS = { + "PKT_A": [(0, 1), (2000, 3)], + "PKT_B": [(1000, 2), (3000, 4)], # Formatted as '2 degC' and '4 degC' + "REGULAR": [(500, 10), (1500, 20), (2500, 30), (3500, 40)], +} + + +def ts_to_iso(ts_ns): + return datetime.fromtimestamp(ts_ns / 1e9, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +@requires_questdb +class TestTlmApiLatestTsdb: + @pytest.fixture + def base_ns(self, questdb_client, clean_table, wait_for_data): + # Align to microseconds since ts_to_iso drops nanoseconds and could round the range start past the first row + base_ns = time.time_ns() // 1000 * 1000 + for packet_name, rows in ROWS.items(): + table_name, _ = QuestDBClient.sanitize_table_name(TARGET, packet_name) + clean_table(table_name) + questdb_client.create_table(TARGET, packet_name, PACKET_DEFS[packet_name]) + for i, (offset_ms, value) in enumerate(rows): + columns = questdb_client.process_json_data({"VALUE": value}, table_name) + columns["RECEIVED_COUNT"] = i + if packet_name == "PKT_B": + columns["VALUE__F"] = f"{value} degC" + questdb_client.write_row(table_name, columns, base_ns + offset_ms * 1_000_000) + questdb_client.flush() + for packet_name, rows in ROWS.items(): + table_name, _ = QuestDBClient.sanitize_table_name(TARGET, packet_name) + assert wait_for_data(questdb_client, table_name, len(rows)) >= len(rows) + with ( + patch.object(TargetModel, "packet", side_effect=lambda _tgt, packet_name, **_kw: PACKET_DEFS[packet_name]), + patch.object(TargetModel, "get_item_to_packet_map", return_value={"VALUE": ["PKT_A", "PKT_B"]}), + patch("openc3.api.tlm_api.authorize"), + ): + yield base_ns + QuestDBClient.disconnect() + + def at(self, base_ns, offset_ms): + return ts_to_iso(base_ns + offset_ms * 1_000_000) + + def test_returns_the_packet_received_most_recently_at_a_point_in_time(self, base_ns): + assert get_tlm_values([f"{TARGET}__LATEST__VALUE__RAW"], start_time=self.at(base_ns, 2500)) == [[3, None]] + + def test_returns_a_packet_received_after_the_last_row_of_the_first_candidate(self, base_ns): + # PKT_A was last received at 2000 but PKT_B at 3000 so PKT_B is the latest at 3500 + assert get_tlm_values([f"{TARGET}__LATEST__VALUE__RAW"], start_time=self.at(base_ns, 3500)) == [[4, None]] + + def test_returns_formatted_from_a_candidate_with_a_formatted_value(self, base_ns): + vals = get_tlm_values([f"{TARGET}__LATEST__VALUE__FORMATTED"], start_time=self.at(base_ns, 3500)) + assert vals == [["4 degC", None]] + + def test_returns_raw_from_a_candidate_without_a_formatted_value(self, base_ns): + assert get_tlm_values([f"{TARGET}__LATEST__VALUE__FORMATTED"], start_time=self.at(base_ns, 2500)) == [[3, None]] + + def test_returns_formatted_from_each_candidate_as_available_in_a_range(self, base_ns): + vals = get_tlm_values( + [f"{TARGET}__LATEST__VALUE__FORMATTED", f"{TARGET}__REGULAR__VALUE__RAW"], + start_time=self.at(base_ns, 0), + end_time=self.at(base_ns, 4000), + ) + assert [row[0][0] for row in vals] == [1, "2 degC", 3, "4 degC"] + + def test_returns_latest_alongside_a_regular_item_at_a_point_in_time(self, base_ns): + vals = get_tlm_values( + [f"{TARGET}__LATEST__VALUE__RAW", f"{TARGET}__REGULAR__VALUE__RAW"], start_time=self.at(base_ns, 3600) + ) + assert vals == [[4, None], [40, None]] + + def test_returns_the_latest_packet_at_each_row_of_a_range_driven_by_a_regular_item(self, base_ns): + vals = get_tlm_values( + [f"{TARGET}__LATEST__VALUE__RAW", f"{TARGET}__REGULAR__VALUE__RAW"], + start_time=self.at(base_ns, 0), + end_time=self.at(base_ns, 4000), + ) + assert vals == [ + [[1, None], [10, None]], + [[2, None], [20, None]], + [[3, None], [30, None]], + [[4, None], [40, None]], + ] diff --git a/openc3/test/integration/tsdb/ruby/tlm_api_latest_tsdb_spec.rb b/openc3/test/integration/tsdb/ruby/tlm_api_latest_tsdb_spec.rb new file mode 100644 index 0000000000..b31c9bc93f --- /dev/null +++ b/openc3/test/integration/tsdb/ruby/tlm_api_latest_tsdb_spec.rb @@ -0,0 +1,109 @@ +# Copyright 2026 OpenC3, Inc. +# All Rights Reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE.md for more details. + +# This file may also be used under the terms of a commercial license +# if purchased from OpenC3, Inc. + +# End-to-end tests for historical get_tlm_values with LATEST items. +# +# LATEST must return the value from whichever packet containing the item was +# received most recently at the requested time, not the packet received most +# recently now, and not just the first packet in the QuestDB ASOF JOIN. +# +# Run with: +# 1. Start QuestDB: docker compose -f docker-compose.test.yml up -d +# 2. Run tests: bundle exec rspec ruby/tlm_api_latest_tsdb_spec.rb +# 3. Stop QuestDB: docker compose -f docker-compose.test.yml down + +require_relative 'spec_helper' +require 'openc3/api/tlm_api' +require 'openc3/utilities/authorization' + +RSpec.describe 'get_tlm_values with historical LATEST', :questdb do + class LatestApi + include OpenC3::Api + include OpenC3::Authorization + end + + # Timeline in ms from the base time (VALUE written to each packet): + # PKT_A: 0 => 1, 2000 => 3 + # PKT_B: 1000 => 2, 3000 => 4 (with units so formatted as '2 degC' and '4 degC') + # REGULAR: 500 => 10, 1500 => 20, 2500 => 30, 3500 => 40 + before(:all) do + @test_params = write_multi_table_data({ + 'tables' => [ + { 'target' => 'RUBY_LT', 'packet' => 'PKT_A', 'data_type' => 'INT', 'bit_size' => 32, + 'rows' => [{ 'offset_ms' => 0, 'values' => { 'VALUE' => 1 } }, { 'offset_ms' => 2000, 'values' => { 'VALUE' => 3 } }] }, + # Only PKT_B has units so only it has a formatted (VALUE__F) column + { 'target' => 'RUBY_LT', 'packet' => 'PKT_B', 'data_type' => 'INT', 'bit_size' => 32, 'units' => 'degC', + 'rows' => [{ 'offset_ms' => 1000, 'values' => { 'VALUE' => 2 }, 'formatted_values' => { 'VALUE' => '2 degC' } }, + { 'offset_ms' => 3000, 'values' => { 'VALUE' => 4 }, 'formatted_values' => { 'VALUE' => '4 degC' } }] }, + { 'target' => 'RUBY_LT', 'packet' => 'REGULAR', 'data_type' => 'INT', 'bit_size' => 32, + 'rows' => [500, 1500, 2500, 3500].each_with_index.map { |offset, i| { 'offset_ms' => offset, 'values' => { 'VALUE' => (i + 1) * 10 } } } }, + ] + }) + end + + before(:each) do + OpenC3::QuestDBClient.disconnect + expect(@test_params['success']).to be true + packet_defs = @test_params['tables'].to_h { |table| [table['packet_name'], table['packet_def']] } + allow(OpenC3::TargetModel).to receive(:packet) { |_target_name, packet_name, **| packet_defs.fetch(packet_name) } + allow(OpenC3::TargetModel).to receive(:get_item_to_packet_map).and_return({ 'VALUE' => ['PKT_A', 'PKT_B'] }) + @api = LatestApi.new + allow(@api).to receive(:authorize) + end + + after(:each) do + OpenC3::QuestDBClient.disconnect + end + + def time_at(offset_ms) + ns = @test_params['base_time_ns'] + (offset_ms * 1_000_000) + Time.at(ns / 1_000_000_000, ns % 1_000_000_000, :nsec).utc.strftime('%Y-%m-%dT%H:%M:%S.%6NZ') + end + + it 'returns the packet received most recently at a point in time' do + expect(@api.get_tlm_values(['RUBY_LT__LATEST__VALUE__RAW'], start_time: time_at(2500))).to eq([[3, nil]]) + end + + it 'returns a packet received after the last row of the first candidate' do + # PKT_A was last received at 2000 but PKT_B at 3000 so PKT_B is the latest at 3500 + expect(@api.get_tlm_values(['RUBY_LT__LATEST__VALUE__RAW'], start_time: time_at(3500))).to eq([[4, nil]]) + end + + it 'returns FORMATTED from a candidate with a formatted value' do + expect(@api.get_tlm_values(['RUBY_LT__LATEST__VALUE__FORMATTED'], start_time: time_at(3500))).to eq([['4 degC', nil]]) + end + + it 'returns RAW from a candidate without a formatted value' do + expect(@api.get_tlm_values(['RUBY_LT__LATEST__VALUE__FORMATTED'], start_time: time_at(2500))).to eq([[3, nil]]) + end + + it 'returns FORMATTED from each candidate as available in a range' do + vals = @api.get_tlm_values(['RUBY_LT__LATEST__VALUE__FORMATTED', 'RUBY_LT__REGULAR__VALUE__RAW'], + start_time: time_at(0), end_time: time_at(4000)) + expect(vals.map { |row| row[0][0] }).to eq([1, '2 degC', 3, '4 degC']) + end + + it 'returns LATEST alongside a regular item at a point in time' do + vals = @api.get_tlm_values(['RUBY_LT__LATEST__VALUE__RAW', 'RUBY_LT__REGULAR__VALUE__RAW'], start_time: time_at(3600)) + expect(vals).to eq([[4, nil], [40, nil]]) + end + + it 'returns the latest packet at each row of a range driven by a regular item' do + vals = @api.get_tlm_values(['RUBY_LT__LATEST__VALUE__RAW', 'RUBY_LT__REGULAR__VALUE__RAW'], + start_time: time_at(0), end_time: time_at(4000)) + expect(vals).to eq([ + [[1, nil], [10, nil]], + [[2, nil], [20, nil]], + [[3, nil], [30, nil]], + [[4, nil], [40, nil]], + ]) + end +end