From f13424ecdcb22cb7910ed415a96385961c06de5b Mon Sep 17 00:00:00 2001 From: mpoemsl <37959974+mpoemsl@users.noreply.github.com> Date: Thu, 16 Mar 2023 15:25:19 +0100 Subject: [PATCH 1/6] Add prometheus queue count metric --- .../pytorch/serve/grpcimpl/InferenceImpl.java | 3 +- .../serve/metrics/api/MetricAggregator.java | 7 +++- .../prometheous/PrometheusMetricManager.java | 33 +++++++++++++++++++ .../java/org/pytorch/serve/util/ApiUtils.java | 3 ++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/grpcimpl/InferenceImpl.java b/frontend/server/src/main/java/org/pytorch/serve/grpcimpl/InferenceImpl.java index 7b613fb121..80b6a3b0a4 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/grpcimpl/InferenceImpl.java +++ b/frontend/server/src/main/java/org/pytorch/serve/grpcimpl/InferenceImpl.java @@ -92,7 +92,6 @@ public void predictions( new InputParameter(entry.getKey(), entry.getValue().toByteArray())); } - MetricAggregator.handleInferenceMetric(modelName, modelVersion); Job job = new GRPCJob( responseObserver, @@ -109,6 +108,8 @@ public void predictions( InternalServerException e = new InternalServerException(responseMessage); sendErrorResponse( responseObserver, Status.INTERNAL, e, "InternalServerException.()"); + } else { + MetricAggregator.handleInferenceMetric(modelName, modelVersion); } } catch (ModelNotFoundException | ModelVersionNotFoundException e) { sendErrorResponse(responseObserver, Status.INTERNAL, e, null); diff --git a/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java b/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java index 383942508f..d6da8f1bdf 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java +++ b/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java @@ -7,14 +7,18 @@ public final class MetricAggregator { private MetricAggregator() {} + // Is executed upon successful Job insertion in queue public static void handleInferenceMetric(final String modelName, final String modelVersion) { ConfigManager configMgr = ConfigManager.getInstance(); if (configMgr.isMetricApiEnable() && configMgr.getMetricsFormat().equals(ConfigManager.METRIC_FORMAT_PROMETHEUS)) { - PrometheusMetricManager.getInstance().incInferCount(modelName, modelVersion); + PrometheusMetricManager metrics = PrometheusMetricManager.getInstance(); + metrics.incInferCount(modelName, modelVersion); + metrics.incQueueCount(modelName, modelVersion); } } + // Is executed upon successful Job completion public static void handleInferenceMetric( final String modelName, final String modelVersion, long timeInQueue, long inferTime) { ConfigManager configMgr = ConfigManager.getInstance(); @@ -23,6 +27,7 @@ public static void handleInferenceMetric( PrometheusMetricManager metrics = PrometheusMetricManager.getInstance(); metrics.incInferLatency(inferTime, modelName, modelVersion); metrics.incQueueLatency(timeInQueue, modelName, modelVersion); + metrics.decQueueCount(modelName, modelVersion); } } } diff --git a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java index 27bc015420..030d1bf92a 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java @@ -1,6 +1,7 @@ package org.pytorch.serve.metrics.format.prometheous; import io.prometheus.client.Counter; +import io.prometheus.client.Gauge; import java.util.UUID; public final class PrometheusMetricManager { @@ -10,6 +11,7 @@ public final class PrometheusMetricManager { private Counter inferRequestCount; private Counter inferLatency; private Counter queueLatency; + private Gauge queueRequestCount; private PrometheusMetricManager() { String[] metricsLabels = {"uuid", "model_name", "model_version"}; @@ -31,6 +33,12 @@ private PrometheusMetricManager() { .labelNames(metricsLabels) .help("Cumulative queue duration in microseconds") .register(); + queueRequestCount = + Gauge.build() + .name("ts_queue_requests_total") + .labelNames(metricsLabels) + .help("Current queue inference request count") + .register(); } private static String getOrDefaultModelVersion(String modelVersion) { @@ -78,4 +86,29 @@ public void incInferCount(String modelName, String modelVersion) { .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion)) .inc(); } + + + /** + * Counts a valid inference request that has been added to a queue + * + * @param modelName name of the model + * @param modelVersion version of the model + */ + public void incQueueCount(String modelName, String modelVersion) { + queueRequestCount + .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion)) + .inc(); + } + + /** + * Counts a valid inference request that has been removed from a queue + * + * @param modelName name of the model + * @param modelVersion version of the model + */ + public void decQueueCount(String modelName, String modelVersion) { + queueRequestCount + .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion)) + .dec(); + } } diff --git a/frontend/server/src/main/java/org/pytorch/serve/util/ApiUtils.java b/frontend/server/src/main/java/org/pytorch/serve/util/ApiUtils.java index c485b67eb7..f808321f48 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/util/ApiUtils.java +++ b/frontend/server/src/main/java/org/pytorch/serve/util/ApiUtils.java @@ -30,6 +30,7 @@ import org.pytorch.serve.http.messages.RegisterModelRequest; import org.pytorch.serve.job.RestJob; import org.pytorch.serve.snapshot.SnapshotManager; +import org.pytorch.serve.metrics.api.MetricAggregator; import org.pytorch.serve.util.messages.RequestInput; import org.pytorch.serve.util.messages.WorkerCommands; import org.pytorch.serve.wlm.Model; @@ -397,6 +398,8 @@ public static RestJob addRESTInferenceJob( int priority = job.getPriority(); String responseMessage = getInferenceErrorResponseMessage(modelName, version, priority); throw new ServiceUnavailableException(responseMessage); + } else { + MetricAggregator.handleInferenceMetric(modelName, version); } return job; } From 1ff2ee4b1cc17f70481a3260330594089bb5a30e Mon Sep 17 00:00:00 2001 From: mpoemsl <37959974+mpoemsl@users.noreply.github.com> Date: Thu, 16 Mar 2023 16:25:08 +0100 Subject: [PATCH 2/6] Fix queue count updating timing --- .../serve/http/api/rest/InferenceRequestHandler.java | 3 --- .../server/src/main/java/org/pytorch/serve/job/GRPCJob.java | 5 +++++ .../server/src/main/java/org/pytorch/serve/job/RestJob.java | 5 +++-- .../metrics/format/prometheous/PrometheusMetricManager.java | 6 +++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/http/api/rest/InferenceRequestHandler.java b/frontend/server/src/main/java/org/pytorch/serve/http/api/rest/InferenceRequestHandler.java index 10774a1206..a3e903ce80 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/http/api/rest/InferenceRequestHandler.java +++ b/frontend/server/src/main/java/org/pytorch/serve/http/api/rest/InferenceRequestHandler.java @@ -21,7 +21,6 @@ import org.pytorch.serve.http.HttpRequestHandlerChain; import org.pytorch.serve.http.ResourceNotFoundException; import org.pytorch.serve.http.StatusResponse; -import org.pytorch.serve.metrics.api.MetricAggregator; import org.pytorch.serve.openapi.OpenApiUtils; import org.pytorch.serve.servingsdk.ModelServerEndpoint; import org.pytorch.serve.util.ApiUtils; @@ -254,8 +253,6 @@ private void predict( NettyUtils.sendJsonResponse(ctx, resp); return; } - - MetricAggregator.handleInferenceMetric(modelName, modelVersion); ApiUtils.addRESTInferenceJob(ctx, modelName, modelVersion, input); } diff --git a/frontend/server/src/main/java/org/pytorch/serve/job/GRPCJob.java b/frontend/server/src/main/java/org/pytorch/serve/job/GRPCJob.java index 3a868f2354..6bb2b0ae59 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/job/GRPCJob.java +++ b/frontend/server/src/main/java/org/pytorch/serve/job/GRPCJob.java @@ -14,6 +14,7 @@ import org.pytorch.serve.http.messages.DescribeModelResponse; import org.pytorch.serve.metrics.Dimension; import org.pytorch.serve.metrics.Metric; +import org.pytorch.serve.metrics.api.MetricAggregator; import org.pytorch.serve.util.ApiUtils; import org.pytorch.serve.util.ConfigManager; import org.pytorch.serve.util.GRPCUtils; @@ -66,6 +67,10 @@ public void response( predictionResponseObserver.onNext(reply); predictionResponseObserver.onCompleted(); + long inferTime = System.nanoTime() - getBegin(); + MetricAggregator.handleInferenceMetric( + getModelName(), getModelVersion(), getScheduled() - getBegin(), inferTime); + logger.debug( "Waiting time ns: {}, Backend time ns: {}", getScheduled() - getBegin(), diff --git a/frontend/server/src/main/java/org/pytorch/serve/job/RestJob.java b/frontend/server/src/main/java/org/pytorch/serve/job/RestJob.java index 5e0d919c04..907838fad7 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/job/RestJob.java +++ b/frontend/server/src/main/java/org/pytorch/serve/job/RestJob.java @@ -136,13 +136,14 @@ private void responseInference( * by external clients. */ if (ctx != null) { - MetricAggregator.handleInferenceMetric( - getModelName(), getModelVersion(), getScheduled() - getBegin(), inferTime); NettyUtils.sendHttpResponse(ctx, resp, true); } else if (responsePromise != null) { responsePromise.complete(body); } + MetricAggregator.handleInferenceMetric( + getModelName(), getModelVersion(), getScheduled() - getBegin(), inferTime); + logger.debug( "Waiting time ns: {}, Backend time ns: {}", getScheduled() - getBegin(), diff --git a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java index 030d1bf92a..0105afc911 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java @@ -25,19 +25,19 @@ private PrometheusMetricManager() { Counter.build() .name("ts_inference_latency_microseconds") .labelNames(metricsLabels) - .help("Cumulative inference duration in microseconds") + .help("Cumulative inference duration in microseconds.") .register(); queueLatency = Counter.build() .name("ts_queue_latency_microseconds") .labelNames(metricsLabels) - .help("Cumulative queue duration in microseconds") + .help("Cumulative queue duration in microseconds.") .register(); queueRequestCount = Gauge.build() .name("ts_queue_requests_total") .labelNames(metricsLabels) - .help("Current queue inference request count") + .help("Current queue inference request count.") .register(); } From f8c3747eee2a335076e67b01b9745d03dba4251f Mon Sep 17 00:00:00 2001 From: mpoemsl <37959974+mpoemsl@users.noreply.github.com> Date: Mon, 20 Mar 2023 16:27:00 +0100 Subject: [PATCH 3/6] Implement per-priority queue count --- .../pytorch/serve/grpcimpl/InferenceImpl.java | 2 +- .../java/org/pytorch/serve/job/GRPCJob.java | 2 +- .../java/org/pytorch/serve/job/RestJob.java | 2 +- .../serve/metrics/api/MetricAggregator.java | 9 ++++--- .../prometheous/PrometheusMetricManager.java | 27 +++++++++++-------- .../java/org/pytorch/serve/util/ApiUtils.java | 2 +- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/grpcimpl/InferenceImpl.java b/frontend/server/src/main/java/org/pytorch/serve/grpcimpl/InferenceImpl.java index 37901cbdcb..3e149d1f5c 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/grpcimpl/InferenceImpl.java +++ b/frontend/server/src/main/java/org/pytorch/serve/grpcimpl/InferenceImpl.java @@ -109,7 +109,7 @@ public void predictions( sendErrorResponse( responseObserver, Status.INTERNAL, e, "InternalServerException.()"); } else { - MetricAggregator.handleInferenceMetric(modelName, modelVersion); + MetricAggregator.handleInferenceMetric(modelName, modelVersion, job.getPriority()); } } catch (ModelNotFoundException | ModelVersionNotFoundException e) { sendErrorResponse(responseObserver, Status.INTERNAL, e, null); diff --git a/frontend/server/src/main/java/org/pytorch/serve/job/GRPCJob.java b/frontend/server/src/main/java/org/pytorch/serve/job/GRPCJob.java index 5d6f8d202f..2dd2c8ed66 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/job/GRPCJob.java +++ b/frontend/server/src/main/java/org/pytorch/serve/job/GRPCJob.java @@ -69,7 +69,7 @@ public void response( long inferTime = System.nanoTime() - getBegin(); MetricAggregator.handleInferenceMetric( - getModelName(), getModelVersion(), getScheduled() - getBegin(), inferTime); + getModelName(), getModelVersion(), getPriority(), getScheduled() - getBegin(), inferTime); logger.debug( "Waiting time ns: {}, Backend time ns: {}", diff --git a/frontend/server/src/main/java/org/pytorch/serve/job/RestJob.java b/frontend/server/src/main/java/org/pytorch/serve/job/RestJob.java index 58ed6293f3..abdbd1483e 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/job/RestJob.java +++ b/frontend/server/src/main/java/org/pytorch/serve/job/RestJob.java @@ -142,7 +142,7 @@ private void responseInference( } MetricAggregator.handleInferenceMetric( - getModelName(), getModelVersion(), getScheduled() - getBegin(), inferTime); + getModelName(), getModelVersion(), getPriority(), getScheduled() - getBegin(), inferTime); logger.debug( "Waiting time ns: {}, Backend time ns: {}", diff --git a/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java b/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java index d6da8f1bdf..df0c9f5208 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java +++ b/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java @@ -2,32 +2,33 @@ import org.pytorch.serve.metrics.format.prometheous.PrometheusMetricManager; import org.pytorch.serve.util.ConfigManager; +import org.pytorch.serve.util.Priority; public final class MetricAggregator { private MetricAggregator() {} // Is executed upon successful Job insertion in queue - public static void handleInferenceMetric(final String modelName, final String modelVersion) { + public static void handleInferenceMetric(final String modelName, final String modelVersion, Priority priority) { ConfigManager configMgr = ConfigManager.getInstance(); if (configMgr.isMetricApiEnable() && configMgr.getMetricsFormat().equals(ConfigManager.METRIC_FORMAT_PROMETHEUS)) { PrometheusMetricManager metrics = PrometheusMetricManager.getInstance(); metrics.incInferCount(modelName, modelVersion); - metrics.incQueueCount(modelName, modelVersion); + metrics.incQueueCount(modelName, modelVersion, priority); } } // Is executed upon successful Job completion public static void handleInferenceMetric( - final String modelName, final String modelVersion, long timeInQueue, long inferTime) { + final String modelName, final String modelVersion, Priority priority, long timeInQueue, long inferTime) { ConfigManager configMgr = ConfigManager.getInstance(); if (configMgr.isMetricApiEnable() && configMgr.getMetricsFormat().equals(ConfigManager.METRIC_FORMAT_PROMETHEUS)) { PrometheusMetricManager metrics = PrometheusMetricManager.getInstance(); metrics.incInferLatency(inferTime, modelName, modelVersion); metrics.incQueueLatency(timeInQueue, modelName, modelVersion); - metrics.decQueueCount(modelName, modelVersion); + metrics.decQueueCount(modelName, modelVersion, priority); } } } diff --git a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java index 0105afc911..0196dbb159 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java @@ -1,7 +1,9 @@ package org.pytorch.serve.metrics.format.prometheous; +import org.pytorch.serve.util.Priority; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; +import java.util.HashMap; import java.util.UUID; public final class PrometheusMetricManager { @@ -11,7 +13,7 @@ public final class PrometheusMetricManager { private Counter inferRequestCount; private Counter inferLatency; private Counter queueLatency; - private Gauge queueRequestCount; + private final HashMap queueRequestCounts; private PrometheusMetricManager() { String[] metricsLabels = {"uuid", "model_name", "model_version"}; @@ -33,12 +35,15 @@ private PrometheusMetricManager() { .labelNames(metricsLabels) .help("Cumulative queue duration in microseconds.") .register(); - queueRequestCount = - Gauge.build() - .name("ts_queue_requests_total") - .labelNames(metricsLabels) - .help("Current queue inference request count.") - .register(); + queueRequestCounts = new HashMap (); + for (Priority priority : Priority.values()) { + queueRequestCounts.put(priority, + Gauge.build() + .name("ts_queue_requests_total") + .labelNames(metricsLabels) + .help("Current queue inference request count.") + .register()); + } } private static String getOrDefaultModelVersion(String modelVersion) { @@ -94,8 +99,8 @@ public void incInferCount(String modelName, String modelVersion) { * @param modelName name of the model * @param modelVersion version of the model */ - public void incQueueCount(String modelName, String modelVersion) { - queueRequestCount + public void incQueueCount(String modelName, String modelVersion, Priority priority) { + queueRequestCounts.get(priority) .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion)) .inc(); } @@ -106,8 +111,8 @@ public void incQueueCount(String modelName, String modelVersion) { * @param modelName name of the model * @param modelVersion version of the model */ - public void decQueueCount(String modelName, String modelVersion) { - queueRequestCount + public void decQueueCount(String modelName, String modelVersion, Priority priority) { + queueRequestCounts.get(priority) .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion)) .dec(); } diff --git a/frontend/server/src/main/java/org/pytorch/serve/util/ApiUtils.java b/frontend/server/src/main/java/org/pytorch/serve/util/ApiUtils.java index a9397b27f3..d8d005c721 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/util/ApiUtils.java +++ b/frontend/server/src/main/java/org/pytorch/serve/util/ApiUtils.java @@ -399,7 +399,7 @@ public static RestJob addRESTInferenceJob( String responseMessage = getInferenceErrorResponseMessage(modelName, version, priority); throw new ServiceUnavailableException(responseMessage); } else { - MetricAggregator.handleInferenceMetric(modelName, version); + MetricAggregator.handleInferenceMetric(modelName, version, job.getPriority()); } return job; } From b57b5b9c5b9dcf41562693ea211e75f1a848f25f Mon Sep 17 00:00:00 2001 From: mpoemsl <37959974+mpoemsl@users.noreply.github.com> Date: Mon, 20 Mar 2023 16:48:37 +0100 Subject: [PATCH 4/6] Add queue size as label to queue metrics --- .../prometheous/PrometheusMetricManager.java | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java index 0196dbb159..6b141cb423 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java @@ -1,8 +1,11 @@ package org.pytorch.serve.metrics.format.prometheous; +import org.pytorch.serve.util.ConfigManager; import org.pytorch.serve.util.Priority; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; +import java.util.Arrays; +import java.util.List; import java.util.HashMap; import java.util.UUID; @@ -10,37 +13,38 @@ public final class PrometheusMetricManager { private static final PrometheusMetricManager METRIC_MANAGER = new PrometheusMetricManager(); private static final String METRICS_UUID = UUID.randomUUID().toString(); - private Counter inferRequestCount; - private Counter inferLatency; - private Counter queueLatency; + private final Counter inferRequestCount; + private final Counter inferLatency; + private final Counter queueLatency; private final HashMap queueRequestCounts; private PrometheusMetricManager() { - String[] metricsLabels = {"uuid", "model_name", "model_version"}; + List metricsLabels = Arrays.asList("uuid", "model_name", "model_version"); inferRequestCount = Counter.build() .name("ts_inference_requests_total") - .labelNames(metricsLabels) + .labelNames(metricsLabels.toArray(new String[0])) .help("Total number of inference requests.") .register(); inferLatency = Counter.build() .name("ts_inference_latency_microseconds") - .labelNames(metricsLabels) + .labelNames(metricsLabels.toArray(new String[0])) .help("Cumulative inference duration in microseconds.") .register(); + metricsLabels.add("priority"); queueLatency = Counter.build() .name("ts_queue_latency_microseconds") - .labelNames(metricsLabels) + .labelNames(metricsLabels.toArray(new String[0])) .help("Cumulative queue duration in microseconds.") .register(); queueRequestCounts = new HashMap (); for (Priority priority : Priority.values()) { queueRequestCounts.put(priority, Gauge.build() - .name("ts_queue_requests_total") - .labelNames(metricsLabels) + .name("ts_queue_requests_" + priority.toString().toLowerCase() + "_total") + .labelNames(metricsLabels.toArray(new String[0])) .help("Current queue inference request count.") .register()); } @@ -75,8 +79,9 @@ public void incInferLatency(long inferTime, String modelName, String modelVersio * @param modelVersion version of the model */ public void incQueueLatency(long queueTime, String modelName, String modelVersion) { + int queueSize = ConfigManager.getInstance().getJobQueueSize(); queueLatency - .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion)) + .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion), String.valueOf(queueSize)) .inc(queueTime / 1000.0); } @@ -100,8 +105,9 @@ public void incInferCount(String modelName, String modelVersion) { * @param modelVersion version of the model */ public void incQueueCount(String modelName, String modelVersion, Priority priority) { + int queueSize = ConfigManager.getInstance().getJobQueueSize(); queueRequestCounts.get(priority) - .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion)) + .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion), String.valueOf(queueSize)) .inc(); } @@ -112,8 +118,9 @@ public void incQueueCount(String modelName, String modelVersion, Priority priori * @param modelVersion version of the model */ public void decQueueCount(String modelName, String modelVersion, Priority priority) { + int queueSize = ConfigManager.getInstance().getJobQueueSize(); queueRequestCounts.get(priority) - .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion)) + .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion), String.valueOf(queueSize)) .dec(); } } From c69a8a0d90af7b5fe3521c24ad093e101c6f08e2 Mon Sep 17 00:00:00 2001 From: mpoemsl <37959974+mpoemsl@users.noreply.github.com> Date: Mon, 20 Mar 2023 17:09:26 +0100 Subject: [PATCH 5/6] Remove list use for metrics labels --- .../prometheous/PrometheusMetricManager.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java index 6b141cb423..5bde233c3b 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java @@ -4,8 +4,6 @@ import org.pytorch.serve.util.Priority; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; -import java.util.Arrays; -import java.util.List; import java.util.HashMap; import java.util.UUID; @@ -19,24 +17,25 @@ public final class PrometheusMetricManager { private final HashMap queueRequestCounts; private PrometheusMetricManager() { - List metricsLabels = Arrays.asList("uuid", "model_name", "model_version"); + String[] metricsLabelsNonQueue = {"uuid", "model_name", "model_version"}; inferRequestCount = Counter.build() .name("ts_inference_requests_total") - .labelNames(metricsLabels.toArray(new String[0])) + .labelNames(metricsLabelsNonQueue) .help("Total number of inference requests.") .register(); inferLatency = Counter.build() .name("ts_inference_latency_microseconds") - .labelNames(metricsLabels.toArray(new String[0])) + .labelNames(metricsLabelsNonQueue) .help("Cumulative inference duration in microseconds.") .register(); - metricsLabels.add("priority"); + + String[] metricsLabelsQueue = {"uuid", "model_name", "model_version", "priority"}; queueLatency = Counter.build() .name("ts_queue_latency_microseconds") - .labelNames(metricsLabels.toArray(new String[0])) + .labelNames(metricsLabelsQueue) .help("Cumulative queue duration in microseconds.") .register(); queueRequestCounts = new HashMap (); @@ -44,7 +43,7 @@ private PrometheusMetricManager() { queueRequestCounts.put(priority, Gauge.build() .name("ts_queue_requests_" + priority.toString().toLowerCase() + "_total") - .labelNames(metricsLabels.toArray(new String[0])) + .labelNames(metricsLabelsQueue) .help("Current queue inference request count.") .register()); } From f9eb9db18676fb29e0d3e32be600b8a915101e36 Mon Sep 17 00:00:00 2001 From: mpoemsl <37959974+mpoemsl@users.noreply.github.com> Date: Mon, 20 Mar 2023 17:36:59 +0100 Subject: [PATCH 6/6] Refactor prometheus queue metrics to have priority and max queue size as labels --- .../serve/metrics/api/MetricAggregator.java | 2 +- .../prometheous/PrometheusMetricManager.java | 32 ++++++++----------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java b/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java index df0c9f5208..dc1f331f3f 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java +++ b/frontend/server/src/main/java/org/pytorch/serve/metrics/api/MetricAggregator.java @@ -27,7 +27,7 @@ public static void handleInferenceMetric( && configMgr.getMetricsFormat().equals(ConfigManager.METRIC_FORMAT_PROMETHEUS)) { PrometheusMetricManager metrics = PrometheusMetricManager.getInstance(); metrics.incInferLatency(inferTime, modelName, modelVersion); - metrics.incQueueLatency(timeInQueue, modelName, modelVersion); + metrics.incQueueLatency(timeInQueue, modelName, modelVersion, priority); metrics.decQueueCount(modelName, modelVersion, priority); } } diff --git a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java index 5bde233c3b..dd9edade21 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/metrics/format/prometheous/PrometheusMetricManager.java @@ -4,7 +4,6 @@ import org.pytorch.serve.util.Priority; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; -import java.util.HashMap; import java.util.UUID; public final class PrometheusMetricManager { @@ -14,7 +13,7 @@ public final class PrometheusMetricManager { private final Counter inferRequestCount; private final Counter inferLatency; private final Counter queueLatency; - private final HashMap queueRequestCounts; + private final Gauge queueRequestCount; private PrometheusMetricManager() { String[] metricsLabelsNonQueue = {"uuid", "model_name", "model_version"}; @@ -31,22 +30,19 @@ private PrometheusMetricManager() { .help("Cumulative inference duration in microseconds.") .register(); - String[] metricsLabelsQueue = {"uuid", "model_name", "model_version", "priority"}; + String[] metricsLabelsQueue = {"uuid", "model_name", "model_version", "priority", "max_queue_size"}; queueLatency = Counter.build() .name("ts_queue_latency_microseconds") .labelNames(metricsLabelsQueue) .help("Cumulative queue duration in microseconds.") .register(); - queueRequestCounts = new HashMap (); - for (Priority priority : Priority.values()) { - queueRequestCounts.put(priority, - Gauge.build() - .name("ts_queue_requests_" + priority.toString().toLowerCase() + "_total") - .labelNames(metricsLabelsQueue) - .help("Current queue inference request count.") - .register()); - } + queueRequestCount = + Gauge.build() + .name("ts_queue_requests_total") + .labelNames(metricsLabelsQueue) + .help("Current queue inference request count.") + .register(); } private static String getOrDefaultModelVersion(String modelVersion) { @@ -77,10 +73,10 @@ public void incInferLatency(long inferTime, String modelName, String modelVersio * @param modelName name of the model * @param modelVersion version of the model */ - public void incQueueLatency(long queueTime, String modelName, String modelVersion) { + public void incQueueLatency(long queueTime, String modelName, String modelVersion, Priority priority) { int queueSize = ConfigManager.getInstance().getJobQueueSize(); queueLatency - .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion), String.valueOf(queueSize)) + .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion), priority.toString(), String.valueOf(queueSize)) .inc(queueTime / 1000.0); } @@ -105,8 +101,8 @@ public void incInferCount(String modelName, String modelVersion) { */ public void incQueueCount(String modelName, String modelVersion, Priority priority) { int queueSize = ConfigManager.getInstance().getJobQueueSize(); - queueRequestCounts.get(priority) - .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion), String.valueOf(queueSize)) + queueRequestCount + .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion), priority.toString(), String.valueOf(queueSize)) .inc(); } @@ -118,8 +114,8 @@ public void incQueueCount(String modelName, String modelVersion, Priority priori */ public void decQueueCount(String modelName, String modelVersion, Priority priority) { int queueSize = ConfigManager.getInstance().getJobQueueSize(); - queueRequestCounts.get(priority) - .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion), String.valueOf(queueSize)) + queueRequestCount + .labels(METRICS_UUID, modelName, getOrDefaultModelVersion(modelVersion), priority.toString(), String.valueOf(queueSize)) .dec(); } }