draft: Updated TableWriter to include service descriptors in query results

cadc-tap-server/build.gradle

@@ -15,7 +15,7 @@ sourceCompatibility = 11
 
 group = 'org.opencadc'
 
-version = '1.1.32'
+version = '1.1.33'
 
 description = 'OpenCADC TAP-1.1 tap server library'
 def git_url = 'https://github.com/opencadc/tap'

cadc-tap-server/src/main/java/ca/nrc/cadc/tap/DefaultTableWriter.java

@@ -96,6 +96,7 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
@@ -140,6 +141,8 @@ public class DefaultTableWriter implements TableWriter {
 
     private static final Map<String, String> knownFormats = new TreeMap<>();
 
+    private transient RegistryClient regClient;
+
     static {
         knownFormats.put(APPLICATION_VOTABLE_XML, VOTABLE);
         knownFormats.put(TEXT_XML, VOTABLE);
@@ -454,7 +457,7 @@ public static URL getAccessURL(String columnID, URI reqStandardID) throws IOExce
     }
 
     // read a votable document matching a {column_id}: currently in the config dir
-    private static VOTableDocument getDoc(String sid) throws IOException {
+    protected static VOTableDocument getDoc(String sid) throws IOException {
         File configDir = new File(System.getProperty("user.home") + "/config");
         String filename = sid + ".xml";
         File tmpl = new File(configDir, filename);
@@ -493,56 +496,62 @@ private static VOTableDocument getDoc(String sid) throws IOException {
      */
     protected void addMetaResources(VOTableDocument votableDocument, List<String> fieldIDs)
             throws IOException {
-        RegistryClient regClient = new RegistryClient();
+
         for (String fid : fieldIDs) {
             VOTableDocument serviceDocument = getDoc(fid);
-            String filename = fid + ".xml";
-            if (serviceDocument == null) {
-                return;
+            if (serviceDocument != null) {
+                for (VOTableResource metaResource : serviceDocument.getResources()) {
+                    if ("meta".equals(metaResource.getType())) {
+                        votableDocument.getResources().add(metaResource);
+                        populateAccessURLParam(metaResource);
+                    }
+                }
             }
+        }
+    }
 
-            for (VOTableResource metaResource : serviceDocument.getResources()) {
-                if ("meta".equals(metaResource.getType())) {
-                    votableDocument.getResources().add(metaResource);
-                    try {
-                        URL accessURL = null;
-                        URI resourceIdentifier = null;
-                        URI standardID = null;
-                        Iterator<VOTableParam> i = metaResource.getParams().iterator();
-                        while (i.hasNext()) {
-                            VOTableParam vp = i.next();
-                            if (vp.getName().equals("accessURL")) {
-                                accessURL = new URL(vp.getValue());
-                            } else if (vp.getName().equals("resourceIdentifier")) {
-                                resourceIdentifier = new URI(vp.getValue());
-                            } else if (vp.getName().equals("standardID")) {
-                                standardID = new URI(vp.getValue());
-                            }
-                        }
-                        if (accessURL == null && resourceIdentifier != null && standardID != null) {
-                            // try to augment resource with accessURL
-                            Subject s = AuthenticationUtil.getCurrentSubject();
-                            AuthMethod cur = AuthenticationUtil.getAuthMethod(s);
-                            if (cur == null) {
-                                cur = AuthMethod.ANON;
-                            }
-                            log.debug("resourceIdentifier=" + resourceIdentifier + ", standardID=" + standardID + ", authMethod=" + cur);
-                            accessURL = regClient.getServiceURL(resourceIdentifier, standardID, cur);
-                            if (accessURL != null) {
-                                String surl = accessURL.toExternalForm();
-                                String arraysize = Integer.toString(surl.length()); // fixed length since we know it
-                                VOTableParam accessParam = new VOTableParam("accessURL", "char", arraysize, surl);
-                                metaResource.getParams().add(accessParam);
-                            } else {
-                                // log the error but continue anyway
-                                log.error("failed to find accessURL: resourceIdentifier=" + resourceIdentifier
-                                        + ", standardID=" + standardID + ", authMethod=" + cur);
-                            }
-                        }
-                    } catch (URISyntaxException e) {
-                        throw new RuntimeException("resourceIdentifier in " + filename + " is invalid", e);
-                    }
+    protected void populateAccessURLParam(VOTableResource metaResource) {
+        if (regClient == null) {
+            regClient = new RegistryClient();
+        }
+        URL accessURL = null;
+        URI resourceIdentifier = null;
+        URI standardID = null;
+        Iterator<VOTableParam> i = metaResource.getParams().iterator();
+        while (i.hasNext()) {
+            VOTableParam vp = i.next();
+            try {
+                if (vp.getName().equals("accessURL")) {
+                    accessURL = new URL(vp.getValue());
+                } else if (vp.getName().equals("resourceIdentifier")) {
+                    resourceIdentifier = new URI(vp.getValue());
+                } else if (vp.getName().equals("standardID")) {
+                    standardID = new URI(vp.getValue());
                 }
+            } catch (URISyntaxException e) {
+                throw new RuntimeException("Something went wrong with the Meta Resource Params", e);
+            } catch (MalformedURLException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        if (accessURL == null && resourceIdentifier != null && standardID != null) {
+            // try to augment resource with accessURL
+            Subject s = AuthenticationUtil.getCurrentSubject();
+            AuthMethod cur = AuthenticationUtil.getAuthMethod(s);
+            if (cur == null) {
+                cur = AuthMethod.ANON;
+            }
+            log.debug("resourceIdentifier=" + resourceIdentifier + ", standardID=" + standardID + ", authMethod=" + cur);
+            accessURL = regClient.getServiceURL(resourceIdentifier, standardID, cur);
+            if (accessURL != null) {
+                String surl = accessURL.toExternalForm();
+                String arraysize = Integer.toString(surl.length()); // fixed length since we know it
+                VOTableParam accessParam = new VOTableParam("accessURL", "char", arraysize, surl);
+                metaResource.getParams().add(accessParam);
+            } else {
+                // log the error but continue anyway
+                log.error("failed to find accessURL: resourceIdentifier=" + resourceIdentifier
+                        + ", standardID=" + standardID + ", authMethod=" + cur);
             }
         }
     }

youcat/build.gradle

@@ -32,7 +32,7 @@ dependencies {
     implementation 'org.opencadc:cadc-uws-server:[1.2.22,)'
     implementation 'org.opencadc:cadc-tap:[1.1.20,)'
     implementation 'org.opencadc:cadc-tap-schema:[1.2.7,)'
-    implementation 'org.opencadc:cadc-tap-server:[1.1.32,)'
+    implementation 'org.opencadc:cadc-tap-server:[1.1.33,)'
     implementation 'org.opencadc:cadc-tap-server-pg:[1.1.1,)'
     implementation 'org.opencadc:cadc-adql:[1.1.4,)'
     implementation 'org.opencadc:cadc-vosi:[1.4.3,2.0)'

youcat/src/intTest/java/org/opencadc/youcat/YoucatTableWriterTest.java

@@ -0,0 +1,186 @@
+package org.opencadc.youcat;
+
+import ca.nrc.cadc.auth.RunnableAction;
+import ca.nrc.cadc.dali.tables.votable.VOTableDocument;
+import ca.nrc.cadc.dali.tables.votable.VOTableReader;
+import ca.nrc.cadc.dali.tables.votable.VOTableResource;
+import ca.nrc.cadc.net.FileContent;
+import ca.nrc.cadc.net.HttpPost;
+import ca.nrc.cadc.net.HttpUpload;
+import ca.nrc.cadc.net.OutputStreamWrapper;
+
+import ca.nrc.cadc.net.ResourceNotFoundException;
+import ca.nrc.cadc.tap.schema.ColumnDesc;
+import ca.nrc.cadc.tap.schema.TableDesc;
+import ca.nrc.cadc.tap.schema.TapDataType;
+import ca.nrc.cadc.tap.schema.TapPermissions;
+import ca.nrc.cadc.util.FileUtil;
+import ca.nrc.cadc.vosi.TableWriter;
+import ca.nrc.cadc.vosi.actions.TablesInputHandler;
+import org.apache.log4j.Logger;
+import org.json.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.security.auth.Subject;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.StringWriter;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+public class YoucatTableWriterTest extends AbstractTablesTest {
+
+    private static final Logger log = Logger.getLogger(YoucatTableWriterTest.class);
+
+    @Test
+    public void testYoucatTableWriter() throws Exception {
+        String tableName = testSchemaName + ".testYoucatWriter";
+
+        createServiceDescriptorTemplates();
+        createCustomTable(tableName);
+
+        String query = "SELECT * FROM " + tableName;
+        List<VOTableResource> resources = executeQueryAndGetResources(query);
+        verifyIDFieldsCount(resources, 3); // c0, c1, c2 are ID fields
+        List<VOTableResource> metaResources = resources.stream().filter(resource -> resource.getType().equals("meta")).collect(Collectors.toList());
+        /*Assert.assertEquals(2, metaResources.size());*/
+
+        query = "SELECT c1, c2, c3, c4, c5 FROM " + tableName;
+        resources = executeQueryAndGetResources(query);
+        verifyIDFieldsCount(resources, 2); // c1, c2 are ID fields
+        metaResources = resources.stream().filter(resource -> resource.getType().equals("meta")).collect(Collectors.toList());
+        /*Assert.assertEquals(1, metaResources.size());*/
+
+        query = "SELECT c2, c3, c4, c5 FROM " + tableName;
+        resources = executeQueryAndGetResources(query);
+        verifyIDFieldsCount(resources, 1); // c2 is an ID field
+        metaResources = resources.stream().filter(resource -> resource.getType().equals("meta")).collect(Collectors.toList());
+        Assert.assertEquals(0, metaResources.size());
+
+        query = "SELECT c3, c4, c5 from " + tableName;
+        resources = executeQueryAndGetResources(query);
+        verifyIDFieldsCount(resources, 0); // no ID fields
+        metaResources = resources.stream().filter(resource -> resource.getType().equals("meta")).collect(Collectors.toList());
+        Assert.assertEquals(0, metaResources.size());
+    }
+
+    private static void verifyIDFieldsCount(List<VOTableResource> resources, long expectedCount) {
+        long fieldsWithIDCount = resources.stream()
+                .filter(resource -> resource.getTable() != null)
+                .flatMap(resource -> resource.getTable().getFields().stream())
+                .filter(field -> field.id != null)
+                .count();
+
+        Assert.assertEquals(expectedCount, fieldsWithIDCount);
+    }
+
+    private void createCustomTable(String tableName) throws Exception {
+        clearSchemaPerms();
+
+        TapPermissions tapPermissions = new TapPermissions(null, true, null, null);
+        super.setPerms(schemaOwner, testSchemaName, tapPermissions, 200);
+
+        doDelete(schemaOwner, tableName, false);
+        TableDesc orig = prepareTable(tableName);
+        uploadTable(tableName, orig);
+
+        log.info("Table " + tableName + " created successfully.");
+    }
+
+    private void createServiceDescriptorTemplates() throws IOException, ResourceNotFoundException {
+        uploadServiceDescriptorTemplate("service-descriptor-template-1.xml", "testServiceDescriptorTemplate1");
+        uploadServiceDescriptorTemplate("service-descriptor-template-2.xml", "testServiceDescriptorTemplate2");
+
+        log.info("Service descriptor templates created successfully.");
+    }
+
+    private void uploadServiceDescriptorTemplate(String sdFileName, String sdName) throws IOException, ResourceNotFoundException {
+        File testFile = FileUtil.getFileFromResource(sdFileName, YoucatTableWriterTest.class);
+        String template = Files.readString(testFile.toPath());
+
+        JSONObject json = new JSONObject();
+        json.put("name", sdName);
+        json.put("template", template);
+
+        FileContent content = new FileContent(json.toString().getBytes(StandardCharsets.UTF_8), "application/json");
+        /*HttpPost httpPost = new HttpPost(templatesURL, content, false); // TODO: use the actual URL for templates when available
+        httpPost.run();
+
+        Assert.assertNull(httpPost.getThrowable());
+        Assert.assertEquals(200, httpPost.getResponseCode());*/
+    }
+
+    private void uploadTable(String tableName, TableDesc orig) throws Exception {
+        URL tableURL = new URL(certTablesURL.toExternalForm() + "/" + tableName);
+        TableWriter w = new TableWriter();
+        StringWriter sw = new StringWriter();
+        w.write(orig, sw);
+        log.info("VOSI-table description:\n" + sw.toString());
+
+        OutputStreamWrapper src = new OutputStreamWrapper() {
+            @Override
+            public void write(OutputStream out) throws IOException {
+                TableWriter w = new TableWriter();
+                w.write(orig, new OutputStreamWriter(out));
+            }
+        };
+        HttpUpload put = new HttpUpload(src, tableURL);
+        put.setContentType(TablesInputHandler.VOSI_TABLE_TYPE);
+        log.info("doCreateTable: " + tableURL);
+        Subject.doAs(schemaOwner, new RunnableAction(put));
+        log.info("doCreateTable: " + put.getResponseCode());
+        if (put.getThrowable() != null && put.getThrowable() instanceof Exception) {
+            throw (Exception) put.getThrowable();
+        }
+        Assert.assertEquals("response code", 200, put.getResponseCode());
+    }
+
+    private TableDesc prepareTable(String tableName) {
+        final TableDesc orig = new TableDesc(testSchemaName, tableName);
+        orig.description = "created by intTest";
+        orig.tableType = TableDesc.TableType.TABLE;
+        orig.tableIndex = 1;
+
+        ColumnDesc c0 = new ColumnDesc(tableName, "c0", TapDataType.STRING);
+        c0.columnID = "testServiceDescriptorID1";
+        orig.getColumnDescs().add(c0);
+
+        ColumnDesc c1 = new ColumnDesc(tableName, "c1", TapDataType.SHORT);
+        c1.columnID = "testServiceDescriptorID2";
+        orig.getColumnDescs().add(c1);
+
+        ColumnDesc c2 = new ColumnDesc(tableName, "c2", TapDataType.INTEGER);
+        c2.columnID = "testServiceDescriptorID3";
+        orig.getColumnDescs().add(c2);
+
+        orig.getColumnDescs().add(new ColumnDesc(tableName, "c3", TapDataType.INTEGER));
+        orig.getColumnDescs().add(new ColumnDesc(tableName, "c4", TapDataType.INTEGER));
+        orig.getColumnDescs().add(new ColumnDesc(tableName, "c5", TapDataType.INTEGER));
+        return orig;
+    }
+
+    private List<VOTableResource> executeQueryAndGetResources(String query) throws Exception {
+        String result = doQuery(query);
+        VOTableReader r = new VOTableReader();
+        VOTableDocument doc = r.read(result);
+
+        return doc.getResources();
+    }
+
+    private String doQuery(String query) throws Exception {
+        Map<String, Object> params = new TreeMap<String, Object>();
+        params.put("LANG", "ADQL");
+        params.put("QUERY", query);
+        String result = Subject.doAs(schemaOwner, new AuthQueryTest.SyncQueryAction(anonQueryURL, params));
+        Assert.assertNotNull(result);
+        return result;
+    }
+}

youcat/src/intTest/resources/service-descriptor-template-1.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<VOTABLE xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.4">
+    <INFO name="name" value="placeholder: will be removed" ID="testServiceDescriptorID1"/>
+    <RESOURCE type="meta" utype="adhoc:service">
+        <PARAM name="standardID" datatype="char" arraysize="*" value="ivo://ivoa.net/std/DataLink#links-1.1" />
+        <PARAM name="accessURL" datatype="char" arraysize="*" value="https://example.net/datalink/links" />
+        <GROUP name="inputParams">
+            <PARAM name="ID" datatype="char" arraysize="*" xtype="uri" ref="testServiceDescriptorID1" value="" />
+        </GROUP>
+    </RESOURCE>
+</VOTABLE>

youcat/src/intTest/resources/service-descriptor-template-2.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<VOTABLE xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.4">
+    <INFO name="name" value="placeholder: will be removed" ID="testServiceDescriptorID2"/>
+    <INFO name="name" value="placeholder: will be removed" ID="testServiceDescriptorID3"/>
+    <RESOURCE type="meta" utype="adhoc:service">
+        <PARAM name="standardID" datatype="char" arraysize="*" value="ivo://ivoa.net/std/DataLink#links-1.1" />
+        <PARAM name="accessURL" datatype="char" arraysize="*" value="https://example.net/datalink/links" />
+        <GROUP name="inputParams">
+            <PARAM name="ID" datatype="char" arraysize="*" xtype="uri" ref="testServiceDescriptorID2" value="" />
+            <PARAM name="ID" datatype="char" arraysize="*" xtype="uri" ref="testServiceDescriptorID3" value="" />
+        </GROUP>
+    </RESOURCE>
+</VOTABLE>