Adding WMS Latency test

README.md

@@ -16,9 +16,12 @@ The CDSE S3 testbed is a Python program designed to interact with a satellite im
 
 The script is designed to be run in a Docker container, as indicated by the Docker commands below. The Docker container provides a controlled environment for the script to run in, ensuring that the performance measurements are consistent across different runs.
 
+5. **WMS Latency Evaluation:** It measures the WMS GetCapabilities and GetMap Requests. 
 
 ## Benchmarking Methodology
 ***
+### **BlueFish.py**
+
 The benchmark is performed using a GDALINFO through the OSGEO module in Python. The Sentinel Band 7 has been selected out of the more common one to limit the risk of pre-caching.
 
 GDALIFO has been selected as it is a common tool used in the geospatial community and it is able to provide a good estimation of the time needed to perform the reads of the metadata data.
@@ -30,6 +33,12 @@ Object storages from the resources are automatically mounted in the docker conta
 The cache, from the client side is disabled, to avoid any bias in the benchmark. 
 As we are aware that some service provider seems to offer a cache system on the server side that cannot be disabled, a meetingation that could be taken is to test each point only once 
 
+### **WMSLatency.py**
+
+The benchark accepts as input a valid CDSE INSTANCE_ID, an output path for the generated reports (csv) and an integer number defining how many times (repeats) the test will be performed. 
+
+- For each repeat the GetCapabilities request is performed and the response time is measured. Then the system sleeps for a random number of seconds (between 2 and 9). For the number of repeats the mean, median, and standrd deviation are computed
+- Similarly the GetMap is measured as well. For each repeat a random layer and random respective style are selected from the available set. The test delivers the same results per layer and in total.
 
 
 ## Prerequisites
@@ -45,4 +54,4 @@ Copy all the files in the `./docker` folder of your system. The `./docker` has t
 - Dockerfile
 - settings
 
-Follow the instructions in the README file to run the CDSE S3 testbed.
+Follow the instructions in the README file to run the CDSE S3 testbed.

WMSLatency.py

@@ -0,0 +1,168 @@
+import numpy as np, os, random, requests, sys, time, xml.etree.ElementTree as ET
+from osgeo import ogr
+ogr.DontUseExceptions()
+
+class WMSLatencyEvaluation(object):
+    def __init__(self, instanceId, validMaskFile, outPath):
+        self._instanceId = instanceId
+        self._baseURL = "https://sh.dataspace.copernicus.eu/ogc/wms/{0}".format(instanceId)
+        self._outPath = outPath
+        self._layers = {}
+        self._epsg3857 = {
+            "minX": -20037508.34,
+            "minY": -20048966.1,
+            "maxX": 20037508.34,
+            "maxY": 20048966.1
+        }
+        inDt = ogr.Open(validMaskFile)
+        inLayer = inDt.GetLayer()
+        self._ft = inLayer.GetNextFeature()
+
+    def __getLayers(self, xmlString):
+        data = ET.fromstring(xmlString)
+        for layer in data.findall(".//{http://www.opengis.net/wms}Layer[@queryable=\'1\']"):
+            namesAndStyles = layer.findall(".//{http://www.opengis.net/wms}Name")
+            self._layers[namesAndStyles[0].text] = []
+            for i in range(1, len(namesAndStyles)):
+                self._layers[namesAndStyles[0].text].append(namesAndStyles[i].text)
+
+
+    def testGetCapabilities(self, repeats=100):
+        params = {
+            "REQUEST": "GetCapabilities",
+            "SERVICE": "WMS",
+            "VERSION": "1.3.0",
+        }
+
+        timeMean = 0
+        timeStDev = 0
+        median = list(range(repeats))
+        response = None
+        for i in range(repeats):
+            response = requests.get(self._baseURL, params=params)
+            tm = response.elapsed.total_seconds()
+            median[i] = tm
+            timeMean += tm/repeats
+            timeStDev += tm*tm/repeats
+            secondsToWait = random.randint(2,9)
+            print("GetCapabilities completed in: {0}, next try in {1} seconds".format( tm, secondsToWait))
+            if (repeats > 1):
+                time.sleep(secondsToWait)
+
+        median.sort()
+        md = median[int(np.floor(repeats/2))]
+        timeStDev = np.sqrt(timeStDev - timeMean*timeMean)
+
+        #csv output file
+        outFl = open(os.path.join(self._outPath,"GetCapabilitiesEvaluation.csv"), "w")
+        outFl.write("Repeats,Mean (s),Median (s),Standard Deviation (s)\n")
+        outFl.write("{0},{1},{2},{3}\n".format(repeats,timeMean,md, timeStDev))
+        outFl.close()
+
+        self.__getLayers(response.text)
+
+    def testGetMap(self, repeats=100, width = 1237, height=589):
+        layerKeys = list(self._layers.keys())
+
+        timeMean = 0
+        timeStDev = 0
+        median = list(range(repeats))
+        response = None
+
+        layerMean = {}
+        layerStdev = {}
+        layerMedian = {}
+
+        for key in layerKeys:
+            layerMean[key] = 0
+            layerStdev[key] = 0
+            layerMedian[key] = []
+
+        for i in range(repeats):
+            #getting an origin within the valid mask
+            inMask = False
+            minX = 0
+            minY = 0
+            while (not inMask):
+                minX = random.uniform(self._epsg3857["minX"], self._epsg3857["maxX"])
+                minY = random.uniform(self._epsg3857["minY"], self._epsg3857["maxY"])
+
+                pnt = ogr.Geometry(ogr.wkbPoint)
+                pnt.SetPoint_2D(0, minX, minY)
+                inMask = self._ft.GetGeometryRef().Intersects(pnt)
+
+            maxX = minX + width*19.2720933430599
+            maxY = minY + height*19.2720933430599
+
+            layerId = random.randint(0,len(self._layers)-1)
+            layerName = layerKeys[layerId]
+            styleId = random.randint(0, len(self._layers[layerName])-1)
+            style  = self._layers[layerName][styleId]
+
+            params = {
+                "REQUEST": "GetMap",
+                "SERVICE": "WMS",
+                "VERSION": "1.3.0",
+                "LAYERS": layerName,
+                "STYLE": style,
+                "FORMAT": "image/png",
+                "DPI":96,
+                "MAP_RESOLUTION":96,
+                "BBOX":"{0},{1},{2},{3}".format(minX, minY, maxX, maxY),
+                "WIDTH":width,
+                "HEIGHT":height,
+            }
+            response = requests.get(self._baseURL, params=params)
+            #print(response.url)
+
+            tm = response.elapsed.total_seconds()
+            median[i] = tm
+            timeMean += tm / repeats
+            timeStDev += tm * tm / repeats
+            secondsToWait = random.randint(2, 9)
+
+            layerMean[layerName] += tm
+            layerStdev[layerName] += tm * tm
+            layerMedian[layerName].append(tm)
+
+            print("GetMap Reqeuest for Layer: {0}, Style: {1}, Completed in {2}s, next try in {3}s".
+                  format(layerName, style, tm, secondsToWait))
+
+            if (repeats > 1):
+                time.sleep(secondsToWait)
+
+        median.sort()
+        timeStDev = np.sqrt(timeStDev - timeMean * timeMean)
+        #per layer stats
+
+        # csv output file
+        outFl = open(os.path.join(self._outPath,"GetMapEvaluation.csv"), "w")
+        outFl.write("Layer,Repeats,Mean (s),Median (s),Standard Deviation (s)\n")
+
+        for layerName in layerKeys:
+            layerrepeats = len(layerMedian[layerName])
+            if layerrepeats > 0:
+                layerMean[layerName] /= layerrepeats
+                layerStdev[layerName] = np.sqrt(layerStdev[layerName]/layerrepeats - layerMean[layerName]*layerMean[layerName])
+                layerMedian[layerName].sort()
+                md = layerMedian[layerName][int(np.floor(layerrepeats / 2))]
+                outFl.write("{0},{1},{2},{3},{4}\n".format(layerName, layerrepeats, layerMean[layerName], md,
+                          layerStdev[layerName]))
+
+
+        outFl.write("Total,{0},{1},{2},{3}\n".format(repeats, timeMean,median[int(np.floor(repeats / 2))],
+                                                                                    timeStDev))
+        outFl.close()
+
+
+def main():
+    if len(sys.argv) < 4:
+        print("usage: python wms_latency_evaluation.py cdse_instance_id valid_mask_file output_report_path repeats")
+        return 1
+
+    obj = WMSLatencyEvaluation(sys.argv[1], sys.argv[2], sys.argv[3])
+    obj.testGetCapabilities(int(sys.argv[4]))
+    obj.testGetMap(int(sys.argv[4]))
+
+if __name__ == "__main__":
+    main()