verbsmarks.proto

// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package verbsmarks.proto;

import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";
import "google/rpc/status.proto";

option java_multiple_files = true;

// ****************************
// Common message types.
// ****************************

// List of specific follower IDs. When it is desired to specify to use specific
// followers in a traffic, list of IDs can be used. The follower IDs are
// generated by the leader, starting from 0, monotonically increasing.
message ParticipantList {
  repeated int32 participant = 1;
}

// Participants specify a subset of followers in the group that participates a
// traffic; it can refer all the followers in the group, a subset with a
// specific size, or list of specific followers.
message Participants {
  oneof participants_oneof {
    // All the followers in the group should participate.
    bool all = 1;

    // A group of any followers of the specified size.
    int32 any_subset = 2;

    // A group of specific follower IDs.
    ParticipantList specific_followers = 3;
  }
}

// ****************************
// Messages to configure the leader.
// ****************************

// All participants send traffic to one of the sinks during the burst cycle and
// stops during rest cycle. Repeats the cycles during the experiment time,
// while changing the sink node every cycle. The sink is chosen in round-robin
// manner.
// With incast, only READ or WRITE operation type should be used and they should
// not be mixed. For READ, the sink should initiate READ to many machines, for
// WRITE, many machines initiate operation to the sink during a short burst of
// time. To coordinate WRITE incast, we use barrier provided by the leader.
message IncastTraffic {
  // The incast victims.
  repeated int32 sinks = 1;

  // The number of participants to generate traffic to the sink.
  int32 degree = 2;

  // Rest period between the burst. Currently, it's the time it can request
  // another barrier operation after one barrier is cleared.
  google.protobuf.Duration rest = 3;

  // The incast traffic sources.
  Participants sources = 4;
}

// Initiators will initiate operations to targets, selecting in uniform random
// manner.
message UniformRandomTraffic {
  Participants initiators = 1;
  Participants targets = 2;

  // Number of QPs to dedicate to each <initiator, target> pair. Defaults
  // to 1 if not specified.
  uint32 qps_per_flow = 3;
}

message ExplicitTraffic {
  message Flow {
    int32 initiator = 1;
    int32 target = 2;

    // Repeats the flow. This is only supported in explicit traffic pattern.
    int32 repeat = 3;
  }

  repeated Flow flows = 1;
  bool bidirectional = 2;
}

// Pingpong traffic is uni-directional; initiators always generate the Ping and
// targets always respond with a Pong. When bidi is desired, use multiple
// traffics. When sample_latency is set, will report last pingpong latency
// samples only, and not distribution stats. If sample_latency is undefined, it
// will be enabled by default if use_event is set.
message PingPongTraffic {
  int32 initiator = 1;
  int32 target = 2;
  int64 iterations = 3;
  bool use_event = 4;
  bool sample_latency = 5;
}

message BandwidthTraffic {
  int32 initiator = 1;
  int32 target = 2;
  bool bidirectional = 3;
  // The number of ops to generate. If set to non-zero number, the experiment
  // will only generate #iterations opsif not reaching experiment_time.
  int64 iterations = 4;
}

enum ConnectionType {
  CONNECTION_TYPE_UNKNOWN = 0;
  CONNECTION_TYPE_RC = 1;
  CONNECTION_TYPE_UD = 2;
}

enum RdmaOp {
  RDMA_OP_UNKNOWN = 0;
  RDMA_OP_SEND_RECEIVE = 1;
  RDMA_OP_READ = 2;
  RDMA_OP_WRITE = 3;
  RDMA_OP_SEND_WITH_IMMEDIATE_RECEIVE = 4;
  RDMA_OP_WRITE_IMMEDIATE = 5;
}

enum ArrivalTimeDistributionType {
  ARRIVAL_TIME_DISTRIBUTION_UNKNOWN = 0;
  ARRIVAL_TIME_DISTRIBUTION_UNIFORM = 1;
  ARRIVAL_TIME_DISTRIBUTION_POISSON = 2;
  ARRIVAL_TIME_DISTRIBUTION_FIXED = 3;
}

// User can provide the intended distribution of the message size. These are
// applied on per QP basis.
enum MessageSizeDistributionType {
  MESSAGE_SIZE_DISTRIBUTION_UNKNOWN = 0;
  MESSAGE_SIZE_DISTRIBUTION_FIXED = 1;

  // Use message size described in RdmaOpRatio.
  MESSAGE_SIZE_DISTRIBUTION_BY_OPS = 2;
}

// User can provide the intended message size using this.
message SizeDistributionParams {
  double mean = 1;  // When the MessageSizeDistributionType is
                    // FIXED_SIZE, `mean` will be used as the exact
                    // message size and other fields will be ignored.

  double min = 2;
  double max = 3;
}

// By repeating this message, users can specify the desired operation and the
// ratio. Each op will have ratio of {op_ratio.ratio / sum of op_ratio.ratio}.
// Example1 ]
// {{op_code: send, ratio: 1}, {op_code: read, ratio: 3}}
// --> 25% send and 75% read operations.
// Example2 ]
// {{op_code: send, ratio: 0.2}, {op_code: read, ratio: 0.8}}
// --> 20% send and 80% read operations.
// If MESSAGE_SIZE_DISTRIBUTION_BY_OPS is selected for size distribution,
// the op size described in this message will be used.
message RdmaOpRatio {
  RdmaOp op_code = 1;

  // The ratio in frequency of issuing this op.
  double ratio = 2;

  // Only applicable when MESSAGE_SIZE_DISTRIBUTION_BY_OPS is chosen.
  int32 op_size = 3;

  // If present, only the followers in the list shall initiate this op.
  repeated int32 initiators = 4;

  // Offered load ratio for this op type. This value is calculated and written
  // by the leader; not user-provided.
  double expected_offered_load_ratio = 5;
}

enum ThreadAffinityType {
  // Do not pin.
  THREAD_AFFINITY_TYPE_NONE = 0;

  // Pin on the CPU_NUM (traffic_pattern_id % CPU count). Proceed when it fails
  // to pin.
  THREAD_AFFINITY_PER_TRAFFIC_PATTERN = 1;

  // Pin on the CPU_NUM (traffic_pattern_id % CPU count). Halt when it fails to
  // pin.
  THREAD_AFFINITY_PER_TRAFFIC_PATTERN_STRICT = 2;

  // Pin the poller on the ((traffic_pattern_id * 2) % CPU count) and the poster
  // on the ((traffic_pattern_id * 2 +1) % CPU count). Proceed when it fails to
  // pin.
  THREAD_AFFINITY_TRAFFIC_PATTERN_SEPARATE = 3;

  // Pin the poller on the ((traffic_pattern_id * 2) % CPU count) and the poster
  // on the ((traffic_pattern_id * 2 +1) % CPU count). Halt when it fails to
  // pin.
  THREAD_AFFINITY_TRAFFIC_PATTERN_SEPARATE_STRICT = 4;

  // Requires ThreadAffinityParameter. Round robin between cpu_min and cpu_max
  // assigned to the follower.
  THREAD_AFFINITY_PER_FOLLOWER_AND_TRAFFIC_PATTERN = 5;

  // Explicitly specify the CPU core to pin. Require ThreadAffinityParameter.
  THREAD_AFFINITY_EXPLICIT = 6;
}

// The proto allows us to explicitly specify {follower_id, traffic_pattern_id}
// pin to which CPU core ID.
message ExplicitThreadAffinityParameter {
  message ExplicitThreadAffinityConfig {
    int32 follower_id = 1;

    int32 traffic_pattern_id = 2;

    int32 cpu_core_id = 3;
  }

  repeated ExplicitThreadAffinityConfig explicit_thread_affinity_configs = 1;
}

// Optional Round-Robin thread affinity parameter.
message ThreadAffinityParameter {
  // When THREAD_AFFINITY_PER_FOLLOWER_AND_TRAFFIC_PATTERN is used, each
  // follower can be pinned to only CPU numbers >=follower_id_to_cpu_min and <
  // follower_id_to_cpu_max. For example, if there are 4 traffic patterns, 2
  // followers, and the user provided the following config
  // follower_id_to_cpu_min[0] = 0
  // follower_id_to_cpu_min[1] = 2
  // follower_id_to_cpu_max[0] = 2
  // follower_id_to_cpu_min=[1] = 4
  // The follower0 will assign traffic0 to core0, traffic1 core1, traffic2
  // to core0, traffic3 to core1. The follower1 will assign its traffic0 to
  // core2, traffic1 to core3, traffic2 to core2, traffic3 to core3.
  map<int32, int32> follower_id_to_cpu_min = 1;
  map<int32, int32> follower_id_to_cpu_max = 2;
}

message TrafficCharacteristics {
  reserved 1, 5, 6, 9, 11;

  // Each op will have ratio of {op_ratio.ratio / sum of op_ratio.ratio}.
  repeated RdmaOpRatio op_ratio = 2;

  // The type of message size distribution. e.g. fixed, normal, etc.
  MessageSizeDistributionType message_size_distribution_type = 3;

  // The size of the message, when applicable: mean, max, etc.
  SizeDistributionParams size_distribution_params_in_bytes = 4;

  // Wait before starting experiments of the particular traffic. Used to align
  // traffics. e.g. all followers are participating in an incast from the
  // beginning, and later two of them gets to have an explicit traffic. The
  // later traffic can be delayed by using this field.
  google.protobuf.Duration delay_time = 7;

  // The amount of time during which to take experiment measurements.
  google.protobuf.Duration experiment_time = 8;

  // The amount of time to wait after starting the experiment before getting
  // measurements, and the amount of time before the end of the experiment to
  // stop taking measurements. This should ensure that all measurements are
  // taken at "steady state".
  google.protobuf.Duration buffer_time = 10;

  // The number of operations that should be submitted in each call to
  // `ibv_post_send`.
  int32 batch_size = 12;

  // If the data is above this size (in bytes), inline the data. The experiment
  // will halt with an error if the threshold is bigger than the max provided by
  // the device.
  int32 inline_threshold = 13;

  // If true, only the last OP in a batch will be signalled and the others are
  // unsignalled.
  bool signal_only_last_op_in_batch = 14;

  // The number of receive requests to prepost.
  int32 num_prepost_receive_ops = 15;

  // If bigger than 0, limit the number of OPs to initiate within this limit.
  // E.g. if this value is set to 3, the follower will only initiate 3 ops and
  // polls or receives during the remaining experiments.
  int32 max_ops_to_initiate = 16;

  // If bigger than 0, verbsmarks will check the result and fail the test if
  // there are not enough successful completions.
  int32 required_completions_for_success = 17;

  // Optional traffic class to set ibv_global_route.traffic_class.
  int32 traffic_class = 18;

  // Use to set mod_rtr.min_rnr_timer.
  uint32 min_rnr_timer = 19;

  // Use to set mod_rtr.timeout.
  uint32 timeout = 20;
}

// Allocation policy of ibverbs protection domains.
enum PdAllocationPolicy {
  PD_ALLOCATION_POLICY_UNKNOWN = 0;

  // Each QP has a distinct protection domain.
  PD_PER_QP = 1;

  // Each traffic pattern has a distinct PD. MRs and QPs in a traffic pattern
  // are all created under this PD.
  PD_PER_TRAFFIC_PATTERN = 2;

  // Each follower allocates only one PD. All its MRs and QPs are all created
  // under this PD.
  PD_PER_FOLLOWER = 3;
}

// How a QP should associated to ibv memory regions.
enum QpMrMapping {
  QP_MR_MAPPING_UNKNOWN = 0;
  QP_HAS_DEDICATED_MRS = 1;
  QP_USES_MRS_IN_PD = 2;
}

// MemoryResourcePolicy defines how protection domains, memory regions and
// memory blocks should be allocated and associated with each other.
message MemoryResourcePolicy {
  PdAllocationPolicy pd_allocation_policy = 1;
  QpMrMapping qp_mr_mapping = 2;

  // For QP_HAS_DEDICATED_MRS, num_mrs_per_qp must be greater than 0; for
  // QP_USES_MRS_IN_PD, num_mrs_per_pd must be greater than 0.
  oneof num_mrs_oneof {
    int32 num_mrs_per_qp = 3;
    int32 num_mrs_per_pd = 4;
  }
}

// Completion queue assignment policy defines the allocation of completion
// queues within a traffic generator and their mapping to queue pairs.
// Currently supports one CQ per traffic generator, shared by all QPs, or
// a private CQ for each QP.
enum CompletionQueuePolicy {
  COMPLETION_QUEUE_POLICY_UNSPECIFIED = 0;
  COMPLETION_QUEUE_POLICY_SHARED_CQ = 1;
  COMPLETION_QUEUE_POLICY_PRIVATE_CQ = 2;
}

// Metrics collection options. If unspecified, default behavior is collecting
// cumulative throughput and latency tdigests, with all optional settings false.
message MetricsCollection {
  // Enable per-second throughput/latency calculation.
  bool enable_per_second_stats = 1;

  // Report detailed latency timestamps for a sequence of operations for each
  // initiator traffic generator.
  bool enable_tracing = 2;
  // Number of iterations to run before tracing operation latency, if
  // `enable_tracing` is set.
  uint32 tracing_warmup_iterations = 3;
  // Disable latency sample buffering for tdigest calculation, reducing memory
  // usage per QP but increasing CPU cycles per operation.
  bool bypass_tdigest_buffering = 4;
}

// GlobalTrafficPattern defines a leader-level traffic pattern. The leader
// generates multiple PerFollowerTrafficPattern for each follower based on this
// information.
message GlobalTrafficPattern {
  int32 global_traffic_id = 1;  // Filled by the leader.

  Participants participants = 2;
  TrafficCharacteristics traffic_characteristics = 3;
  ConnectionType connection_type = 4;

  reserved 5;

  // When true, writes new data to the buffer for every send or write operation.
  // When false, only set the data of the memory block when the queue pair is
  // initialized.
  bool populate_op_buffers = 6;

  message OpenLoopParameters {
    double offered_load_gbps = 1;
    ArrivalTimeDistributionType arrival_time_distribution_type = 2;
  }

  // Specifies the method with which operation load will be determined, either
  // open loop or closed loop.
  oneof load_parameters {
    OpenLoopParameters open_loop_parameters = 7;
    int32 closed_loop_max_outstanding = 8;
  }

  // If true, check buffer memory contents for expected data pattern.
  bool validate_op_buffers = 9;

  // If bigger than 0, used for queue depth. If 0, verbsmarks decides what
  // value should be picked based on the traffic requirement.
  int32 override_queue_depth = 10;

  // Parameters specific to each traffic type.
  oneof traffic_oneof {
    IncastTraffic incast_traffic = 100;
    UniformRandomTraffic uniform_random_traffic = 101;
    ExplicitTraffic explicit_traffic = 102;
    PingPongTraffic pingpong_traffic = 103;
    BandwidthTraffic bandwidth_traffic = 104;
  }

  // Specifies the completion queue assignment policy within each traffic
  // generator. If unspecified, each QP will utilize a private CQ.
  CompletionQueuePolicy completion_queue_policy = 105;

  // Specifies metrics to be collected by each follower. If unspecified, all
  // throughput and latency metrics will be calculated.
  MetricsCollection metrics_collection = 106;
}

// Configuration for the leader, including global traffic pattern.
message LeaderConfig {
  // The size of the group. There is only one group. A participant might belong
  // to more than one traffic, and the traffic description will include which
  // followers should participate that traffic using follower ID.
  int32 group_size = 1;

  // Time limit for forming a quorum of followers in seconds.
  int32 quorum_time_out_sec = 2;

  // Set of traffic patterns. This allows composition of traffic patterns.
  // For example, to support uniform-random traffic and incast at the same time,
  // there should be one GlobalTrafficPattern for uni-random and one
  // GlobalTrafficPattern for incast.
  repeated GlobalTrafficPattern traffic_patterns = 3;

  // This optional map contains keys that are follower aliases and values that
  // are follower ids. This provides more control over which machines have which
  // roles in a traffic pattern. If the map is provided, it must contain valid
  // values for all followers in the quorum. If no values are provided, ids will
  // be assigned based on the order in which followers join the quorum.
  map<string, int32> follower_alias_to_id = 4;

  // The optional thread affinity for traffic generator.
  ThreadAffinityType thread_affinity_type = 13;

  // Some ThreadAffinityType needs extra information. Should only provide
  // thread_affinity_param or explicit_thread_affinity_param. Mixed thread
  // affinity config is not allowed.
  ThreadAffinityParameter thread_affinity_param = 14;
  ExplicitThreadAffinityParameter explicit_thread_affinity_param = 15;

  // Optional description about the experiment.
  string description = 16;

  // Optional memory resource policy on how protection domains and memory
  // regions should be allocated.
  MemoryResourcePolicy memory_resource_policy = 17;
}

// ****************************
// Messages for leader - follower communication.
// ****************************

message QueuePairConfig {
  reserved 4, 13;

  int32 peer = 1;
  ConnectionType connection_type = 2;

  // For QueuePair init attributes.
  // The number of maximum outstanding Send or Receive Requests in the
  // QueuePair.
  uint32 max_outstanding_ops = 3;

  // When true, sets the op buffer data deterministically so that it can be
  // validated that the correct data was set inter-process or inter-machine.
  bool validate_op_buffers = 5;

  // Unique within a traffic pattern.
  int32 queue_pair_id = 6;

  // When true, this queue pair triggers Read/Write/Send operations.
  bool is_initiator = 7;

  // When true, this queue pair is the destination of Read/Write/Send operations
  // from its counterpart. Both is_initiator and is_target can be true, and at
  // least one of them should be true.
  bool is_target = 8;

  // The maximum size of any operation that will be posted to this queue pair.
  int32 max_op_size = 9;

  // When true, populates the buffer for every operation, according to
  // `validate_op_buffers`, otherwise, populate the buffer only during the
  // initialization.
  bool populate_op_buffers = 10;

  // The number of prepost of receive request. Only applicable when it should
  // post receives.
  int32 num_prepost_receive_ops = 11;

  // If bigger than 0, use this for queue_depth, instead of calculated values.
  uint32 override_queue_depth = 12;

  // Use to set ibv_global_route.traffic_class.
  int32 traffic_class = 14;

  // Use to set mod_rtr.min_rnr_timer.
  uint32 min_rnr_timer = 15;

  // Use to set mod_rtr.timeout.
  uint32 timeout = 16;

  // Specifies metrics to be collected for the queue pair. If unspecified, all
  // throughput and latency metrics will be calculated.
  MetricsCollection metrics_collection = 17;
}

enum TrafficType {
  TRAFFIC_TYPE_UNKNOWN = 0;

  // All the flows are explicitly described.
  TRAFFIC_TYPE_EXPLICIT = 1;

  // Initiators choose targets in uniform random manner.
  TRAFFIC_TYPE_UNIFORM_RANDOM = 2;

  // Following micro burst and rest cycles, many nodes send traffic to the sink
  // during the burst cycle.
  TRAFFIC_TYPE_INCAST = 3;

  // Pingpong
  TRAFFIC_TYPE_PINGPONG = 4;
  TRAFFIC_TYPE_BANDWIDTH = 5;

  // Explicit traffic pattern with one shared completion queue per traffic
  // generator instance.
}

// Information for when thread affinity is used.
message ThreadAffinityInfo {
  // If affinity is used, try to pin the threads on `cpu_to_pin`.
  int32 cpu_to_pin_poller = 1;
  int32 cpu_to_pin_poster = 2;

  // If true, halt when it fails to pin on the requested CPU.
  bool halt_if_fail_to_pin = 3;
}

// PerFollowerTrafficPattern defines a traffic pattern that needs to be
// supported by a follower. A follower can have multiple
// PerFollowerTrafficPattern. It should provide sufficient information for a
// follower to generate the traffic pattern.
message PerFollowerTrafficPattern {
  reserved 5 to 14, 26;
  // Associated global traffic pattern.
  int32 global_traffic_pattern_id = 1;

  // Follower that will execute this traffic pattern.
  int32 follower_id = 2;

  // The peers need to interact with, specified by their IDs.
  ParticipantList peers = 3;
  repeated QueuePairConfig queue_pairs = 4;

  // Overall characteristics that are forwarded from the global config to the
  // per-follower config.
  TrafficCharacteristics traffic_characteristics = 15;

  message OpenLoopParameters {
    // The average message size / offered load. Use this as base when
    // scheduling next op, depending on the distribution type.
    google.protobuf.Duration average_interval = 1;
    ArrivalTimeDistributionType arrival_time_distribution_type = 2;
  }

  // Specifies the method with which operation load will be determined, either
  // open loop or closed loop.
  oneof load_parameters {
    OpenLoopParameters open_loop_parameters = 16;
    int32 closed_loop_max_outstanding = 17;
  }

  // Traffic type. Used in choosing QueuePairSelector. Some traffic types such
  // as incast requires additional information.
  TrafficType traffic_type = 18;

  // Optional per follower initial delay before starting to post.
  google.protobuf.Duration initial_delay_before_posting = 19;
  int32 number_to_post_at_once = 20;
  ThreadAffinityInfo thread_affinity_info = 21;

  // Only applicable for pingpong traffic. If true, acts as a client and sends
  // first. If false, wait until first receive before sending.
  bool is_pingpong_initiator = 22;

  // When not 0, use this instead of time to finish experiment.
  int64 iterations = 23;

  // Specifies the completion queue assignment policy within each traffic
  // generator. If unspecified, each QP will utilize a private CQ.
  CompletionQueuePolicy completion_queue_policy = 24;

  // Use event-driven completion notification, instead of poll-mode. Supported
  // by pingpong traffic type only.
  bool use_event = 25;

  // Specifies metrics to be collected by each follower. If unspecified, all
  // throughput and latency metrics will be calculated.
  MetricsCollection metrics_collection = 27;

  // Parameters specific to each traffic type, copied from GlobalTrafficPattern
  // traffic_oneof field.
  oneof traffic_oneof {
    IncastTraffic incast_traffic = 28;
    UniformRandomTraffic uniform_random_traffic = 29;
    ExplicitTraffic explicit_traffic = 30;
    PingPongTraffic pingpong_traffic = 31;
    BandwidthTraffic bandwidth_traffic = 32;
  }
}

// Information describing a single follower.
message Follower {
  // Leader gives a unique follower ID for each follower.
  fixed32 follower_id = 1;

  // HTTP address for the follower.
  string http_server_address = 2;

  // follower's alias.
  string follower_alias = 3;
}

// ****************************
// Messages for the leader gRPC service.
// ****************************

// Follower->Leader request to join the quorum upon follower initialization.
message QuorumRequest {
  // Each follower reports its HTTP server to the Leader. The Leader announces
  // this address to all followers.
  string http_server_address = 1;

  // Each follower provides an alias for reporting and debugging. If not
  // provided, stringified follower_id to be used.
  string follower_alias = 2;
}

// Leader->Follower response to QuorumRequest. Sent by the leader upon receiving
// JoinQuorum from all expected followers, or upon error.
message QuorumResponse {
  // OK means the quorum reached. Any other code means either the follower
  // could not join the quorum.
  google.rpc.Status status = 5;

  // All followers, including self (follower which sent QuorumRequest).
  repeated Follower followers = 2;
}

// Follower->Leader message to indicate follower is ready to initialize traffic
// patterns. The follower must start ConnectionCoordinator service before
// sending this request.
message ReadyRequest {
  fixed32 follower_id = 1;
}

// Leader->Follower response to ReadyRequest. Leader sends when all followers
// are ready. Follower initializes traffic generators with the provided traffic
// patterns and memory resources, or exits if error status.
message ReadyResponse {
  // Status carries an error if something went wrong globally, e.g. if one of
  // two traffics is not supported, everyone gets failure.
  google.rpc.Status status = 1;
  repeated PerFollowerTrafficPattern per_follower_traffic_patterns = 2;
  MemoryResourcePolicy memory_resource_policy = 3;
}

// Follower->Leader message to signal that the follower is ready to start the
// experiment workload. The follower must finish QueuePair creation before
// sending this message.
message StartExperimentRequest {
  fixed32 follower_id = 1;

  // When a follower fails to prepare for experiment, the error is carried in
  // status.
  google.rpc.Status status = 2;
}

// Leader->Follower response to StartExperimentResponse. Leader sends message
// when all followers are ready to start. Follower starts traffic generation
// upon receiving OK status, or error upon exit.
message StartExperimentResponse {
  // If any follower reported error, we do not start experiment.
  google.rpc.Status status = 1;
  // If set, followers must send heartbeats to leader to communicate health
  // status, and leader will abort if healthy heartbeats are not received from
  // all followers.
  bool enable_heartbeats = 2;
}

// Generic container for measured performance metrics with description label.
message Metric {
  string description = 1;

  oneof value_oneof {
    double double_value = 2;
    google.protobuf.Duration duration_value = 3;
  }
}

message ThroughputResult {
  double ops_per_second = 1;
  double bytes_per_second = 2;
  double gbps = 3;
  int32 seconds_from_start = 4;
}

message LatencyResult {
  google.protobuf.Duration average_latency = 1;
  google.protobuf.Duration median_latency = 2;
  google.protobuf.Duration p99_latency = 3;
  google.protobuf.Duration p999_latency = 4;
  google.protobuf.Duration p9999_latency = 5;
  google.protobuf.Duration min_latency = 6;
  google.protobuf.Duration max_latency = 7;
}

message Statistics {
  ThroughputResult throughput = 1;
  LatencyResult latency = 2;
  int32 outstanding_ops = 3;
}

message StatisticsOverTime {
  repeated Statistics statistics = 1;
}

message PerTrafficResult {
  reserved 7, 9, 10, 11, 15;

  int32 global_traffic_id = 1;
  google.rpc.Status status = 2;
  int64 num_failed_post_operations = 3;
  int64 num_failed_poll_operations = 4;
  int64 num_latency_obtained = 5;
  int64 num_completions = 14;
  int64 num_remaining_outstanding_ops = 16;
  repeated Metric metrics = 6;

  message PerOpTypeSizeResult {
    RdmaOp op_code = 1;
    int32 op_size = 2;
    int64 num_successful_ops = 3;
    ThroughputResult throughput = 4;
    LatencyResult latency = 5;
    StatisticsOverTime statistics_per_second = 6;
  }

  message PerQueuePairResult {
    int64 queue_pair_id = 1;

    reserved 2;

    // Used for debugging information, e.g. how long it takes to post an
    // operation.
    repeated Metric metrics = 3;
    repeated PerOpTypeSizeResult per_op_type_size_results = 4;
  }

  repeated PerQueuePairResult per_queue_pair_results = 8;

  ThroughputResult throughput = 12;
  LatencyResult latency = 13;

  // Used for debugging information. The time that no op finished for this
  // traffic among all queue pairs, in seconds.
  // We are collecting this info since we noticed that some runs missed data
  // from some of the seconds, but the test still passed since there were
  // still some data gets collected. We want to uncover such scenario.
  uint32 no_op_time_sec = 17;

  // Raw latency samples collected from pingpong traffic generator, filled if
  // sampling is enabled via sample_latency field.
  repeated google.protobuf.Duration latency_samples = 18;
}

// Real-time per-second metrics communicated from follower to leader while
// experiment is running. Distinct from PerQueuePairResult, PerOpTypeSizeResult,
// etc., because these do not include cumulative stats.
message CurrentStats {
  message PerOpTypeSizeCurrentStats {
    RdmaOp op_type = 1;
    int32 op_size = 2;
    StatisticsOverTime stats_per_second = 3;
  }

  message QueuePairCurrentStats {
    int32 queue_pair_id = 1;
    repeated PerOpTypeSizeCurrentStats per_op_type_size_current_stats = 2;
  }

  message TrafficCurrentStats {
    int32 global_traffic_id = 1;
    repeated QueuePairCurrentStats queue_pair_current_stats = 3;
  }

  repeated TrafficCurrentStats traffic_current_stats = 1;
}

// Real-time results output generated by leader throughout experiment duration.
message PeriodicResultReport {
  message FollowerCurrentStats {
    int32 follower_id = 1;
    CurrentStats current_stats = 2;
  }
  repeated FollowerCurrentStats follower_current_stats = 1;
}

// The follower sends a summary of the result to the leader when its experiments
// are finished.
message ResultRequest {
  fixed32 follower_id = 1;
  string follower_alias = 6;
  google.rpc.Status status = 2;  // Overall status

  repeated PerTrafficResult per_traffic_results = 3;

  // The average latency of all traffic patterns.
  google.protobuf.Duration average_latency = 4;

  // The total throughput of all traffic patterns.
  ThroughputResult throughput = 5;

  // The approximated median, p99+ latencies of all traffic patterns.
  // Note that this is not the true median, p99, etc, but median of medians,
  // median of p99, etc.
  LatencyResult latency = 7;
}

// The leader sends out this message when it receives ResultRequest from
// everyone. As all the experiments are finished, the followers can destroy
// resources after receiving this message.
message ResultResponse {
  google.rpc.Status status = 1;  // Overall status
}

// Heartbeat sent from follower to leader indicating follower's health status.
message HeartbeatRequest {
  int32 follower_id = 1;
  google.rpc.Status follower_status = 2;
  CurrentStats current_stats = 3;
}

// Heartbeat response from leader to follower. Follower will exit if leader
// sends non-OK status.
message HeartbeatResponse {
  google.rpc.Status leader_status = 1;
  int32 failure_follower_id = 2;
  google.rpc.Status failure_reason = 3;
}

// When the follower needs a barrier for timing coordination, the follower sends
// this request. The decision may be based on the information provided in the
// message, or simply based on the order of the request.
message BarrierRequest {
  int32 global_traffic_id = 1;
  int32 follower_id = 2;
}

// The barrier is cleared upon receiving this message.
message BarrierResponse {
  bool post_granted = 1;
}

// ResultReport combines results reported from all followers, and the associated
// LeaderConfig used to generate the results, in a single message.
message ResultReport {
  LeaderConfig config = 1;
  repeated ResultRequest follower_results = 2;
  repeated Follower followers = 3;

  // End time of overall experiment.
  google.protobuf.Timestamp end_timestamp = 4;

  // Optional user-provided string to record information about the experiment.
  string description = 5;

  // Optional user-provided tags to record information about the experiment.
  repeated string tags = 6;
}

// Verbsmarks gRPC service.
service VerbsMarks {
  // All followers begin their execution with a SeekQuorum RPC to the Leader.
  // The Leader does not respond to SeekQuorum until:
  // -> Enough followers have "Checked in" to establish a usable quorum, and,
  // -> Enough time has elapsed to allow further stragglers to start.
  // Once both of these conditions are met, the Leader responds to all
  // SeekQuorum, and subsequent (late) RPCs are sent an error response.
  rpc SeekQuorum(QuorumRequest) returns (QuorumResponse) {}

  // The follower requests ReadyResponse to the leader.
  // The ReadyResponse should include all traffic pattern information applicable
  // to the follower.
  rpc GetReady(ReadyRequest) returns (ReadyResponse) {}

  // The follower notifies the leader that it finished preparation for
  // experiments.
  rpc StartExperiment(StartExperimentRequest)
      returns (StartExperimentResponse) {}

  // Followers report their result to the leader. Once the leader receives
  // result report from everyone, it will send Result Response. Then the
  // follower can destroy all the resources.
  rpc FinishExperiment(ResultRequest) returns (ResultResponse) {}

  // Followers send heartbeats to the leader on a periodic interval. The leader
  // response with OK/abort status, ending the experiment if a follower fails
  // to send a heartbeat or sends an error status.
  rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse) {}
}