diff --git a/cadc-tap-server/build.gradle b/cadc-tap-server/build.gradle index 91d87c94..6a4b7fd9 100644 --- a/cadc-tap-server/build.gradle +++ b/cadc-tap-server/build.gradle @@ -15,7 +15,7 @@ sourceCompatibility = 11 group = 'org.opencadc' -version = '1.1.32' +version = '1.1.33' description = 'OpenCADC TAP-1.1 tap server library' def git_url = 'https://github.com/opencadc/tap' diff --git a/cadc-tap-server/src/main/java/ca/nrc/cadc/tap/DefaultTableWriter.java b/cadc-tap-server/src/main/java/ca/nrc/cadc/tap/DefaultTableWriter.java index 008313e1..70ced6e8 100644 --- a/cadc-tap-server/src/main/java/ca/nrc/cadc/tap/DefaultTableWriter.java +++ b/cadc-tap-server/src/main/java/ca/nrc/cadc/tap/DefaultTableWriter.java @@ -96,6 +96,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -140,6 +141,8 @@ public class DefaultTableWriter implements TableWriter { private static final Map knownFormats = new TreeMap<>(); + private transient RegistryClient regClient; + static { knownFormats.put(APPLICATION_VOTABLE_XML, VOTABLE); knownFormats.put(TEXT_XML, VOTABLE); @@ -454,7 +457,7 @@ public static URL getAccessURL(String columnID, URI reqStandardID) throws IOExce } // read a votable document matching a {column_id}: currently in the config dir - private static VOTableDocument getDoc(String sid) throws IOException { + protected static VOTableDocument getDoc(String sid) throws IOException { File configDir = new File(System.getProperty("user.home") + "/config"); String filename = sid + ".xml"; File tmpl = new File(configDir, filename); @@ -493,56 +496,62 @@ private static VOTableDocument getDoc(String sid) throws IOException { */ protected void addMetaResources(VOTableDocument votableDocument, List fieldIDs) throws IOException { - RegistryClient regClient = new RegistryClient(); + for (String fid : fieldIDs) { VOTableDocument serviceDocument = getDoc(fid); - String filename = fid + ".xml"; - if (serviceDocument == null) { - return; + if (serviceDocument != null) { + for (VOTableResource metaResource : serviceDocument.getResources()) { + if ("meta".equals(metaResource.getType())) { + votableDocument.getResources().add(metaResource); + populateAccessURLParam(metaResource); + } + } } + } + } - for (VOTableResource metaResource : serviceDocument.getResources()) { - if ("meta".equals(metaResource.getType())) { - votableDocument.getResources().add(metaResource); - try { - URL accessURL = null; - URI resourceIdentifier = null; - URI standardID = null; - Iterator i = metaResource.getParams().iterator(); - while (i.hasNext()) { - VOTableParam vp = i.next(); - if (vp.getName().equals("accessURL")) { - accessURL = new URL(vp.getValue()); - } else if (vp.getName().equals("resourceIdentifier")) { - resourceIdentifier = new URI(vp.getValue()); - } else if (vp.getName().equals("standardID")) { - standardID = new URI(vp.getValue()); - } - } - if (accessURL == null && resourceIdentifier != null && standardID != null) { - // try to augment resource with accessURL - Subject s = AuthenticationUtil.getCurrentSubject(); - AuthMethod cur = AuthenticationUtil.getAuthMethod(s); - if (cur == null) { - cur = AuthMethod.ANON; - } - log.debug("resourceIdentifier=" + resourceIdentifier + ", standardID=" + standardID + ", authMethod=" + cur); - accessURL = regClient.getServiceURL(resourceIdentifier, standardID, cur); - if (accessURL != null) { - String surl = accessURL.toExternalForm(); - String arraysize = Integer.toString(surl.length()); // fixed length since we know it - VOTableParam accessParam = new VOTableParam("accessURL", "char", arraysize, surl); - metaResource.getParams().add(accessParam); - } else { - // log the error but continue anyway - log.error("failed to find accessURL: resourceIdentifier=" + resourceIdentifier - + ", standardID=" + standardID + ", authMethod=" + cur); - } - } - } catch (URISyntaxException e) { - throw new RuntimeException("resourceIdentifier in " + filename + " is invalid", e); - } + protected void populateAccessURLParam(VOTableResource metaResource) { + if (regClient == null) { + regClient = new RegistryClient(); + } + URL accessURL = null; + URI resourceIdentifier = null; + URI standardID = null; + Iterator i = metaResource.getParams().iterator(); + while (i.hasNext()) { + VOTableParam vp = i.next(); + try { + if (vp.getName().equals("accessURL")) { + accessURL = new URL(vp.getValue()); + } else if (vp.getName().equals("resourceIdentifier")) { + resourceIdentifier = new URI(vp.getValue()); + } else if (vp.getName().equals("standardID")) { + standardID = new URI(vp.getValue()); } + } catch (URISyntaxException e) { + throw new RuntimeException("Something went wrong with the Meta Resource Params", e); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + if (accessURL == null && resourceIdentifier != null && standardID != null) { + // try to augment resource with accessURL + Subject s = AuthenticationUtil.getCurrentSubject(); + AuthMethod cur = AuthenticationUtil.getAuthMethod(s); + if (cur == null) { + cur = AuthMethod.ANON; + } + log.debug("resourceIdentifier=" + resourceIdentifier + ", standardID=" + standardID + ", authMethod=" + cur); + accessURL = regClient.getServiceURL(resourceIdentifier, standardID, cur); + if (accessURL != null) { + String surl = accessURL.toExternalForm(); + String arraysize = Integer.toString(surl.length()); // fixed length since we know it + VOTableParam accessParam = new VOTableParam("accessURL", "char", arraysize, surl); + metaResource.getParams().add(accessParam); + } else { + // log the error but continue anyway + log.error("failed to find accessURL: resourceIdentifier=" + resourceIdentifier + + ", standardID=" + standardID + ", authMethod=" + cur); } } } diff --git a/youcat/build.gradle b/youcat/build.gradle index 14567411..34bfa99b 100644 --- a/youcat/build.gradle +++ b/youcat/build.gradle @@ -32,7 +32,7 @@ dependencies { implementation 'org.opencadc:cadc-uws-server:[1.2.22,)' implementation 'org.opencadc:cadc-tap:[1.1.20,)' implementation 'org.opencadc:cadc-tap-schema:[1.2.7,)' - implementation 'org.opencadc:cadc-tap-server:[1.1.32,)' + implementation 'org.opencadc:cadc-tap-server:[1.1.33,)' implementation 'org.opencadc:cadc-tap-server-pg:[1.1.1,)' implementation 'org.opencadc:cadc-adql:[1.1.4,)' implementation 'org.opencadc:cadc-vosi:[1.4.3,2.0)' diff --git a/youcat/src/intTest/java/org/opencadc/youcat/YoucatTableWriterTest.java b/youcat/src/intTest/java/org/opencadc/youcat/YoucatTableWriterTest.java new file mode 100644 index 00000000..eebdb906 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/YoucatTableWriterTest.java @@ -0,0 +1,186 @@ +package org.opencadc.youcat; + +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.dali.tables.votable.VOTableDocument; +import ca.nrc.cadc.dali.tables.votable.VOTableReader; +import ca.nrc.cadc.dali.tables.votable.VOTableResource; +import ca.nrc.cadc.net.FileContent; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.net.HttpUpload; +import ca.nrc.cadc.net.OutputStreamWrapper; + +import ca.nrc.cadc.net.ResourceNotFoundException; +import ca.nrc.cadc.tap.schema.ColumnDesc; +import ca.nrc.cadc.tap.schema.TableDesc; +import ca.nrc.cadc.tap.schema.TapDataType; +import ca.nrc.cadc.tap.schema.TapPermissions; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.vosi.TableWriter; +import ca.nrc.cadc.vosi.actions.TablesInputHandler; +import org.apache.log4j.Logger; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +import javax.security.auth.Subject; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.StringWriter; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + +public class YoucatTableWriterTest extends AbstractTablesTest { + + private static final Logger log = Logger.getLogger(YoucatTableWriterTest.class); + + @Test + public void testYoucatTableWriter() throws Exception { + String tableName = testSchemaName + ".testYoucatWriter"; + + createServiceDescriptorTemplates(); + createCustomTable(tableName); + + String query = "SELECT * FROM " + tableName; + List resources = executeQueryAndGetResources(query); + verifyIDFieldsCount(resources, 3); // c0, c1, c2 are ID fields + List metaResources = resources.stream().filter(resource -> resource.getType().equals("meta")).collect(Collectors.toList()); + /*Assert.assertEquals(2, metaResources.size());*/ + + query = "SELECT c1, c2, c3, c4, c5 FROM " + tableName; + resources = executeQueryAndGetResources(query); + verifyIDFieldsCount(resources, 2); // c1, c2 are ID fields + metaResources = resources.stream().filter(resource -> resource.getType().equals("meta")).collect(Collectors.toList()); + /*Assert.assertEquals(1, metaResources.size());*/ + + query = "SELECT c2, c3, c4, c5 FROM " + tableName; + resources = executeQueryAndGetResources(query); + verifyIDFieldsCount(resources, 1); // c2 is an ID field + metaResources = resources.stream().filter(resource -> resource.getType().equals("meta")).collect(Collectors.toList()); + Assert.assertEquals(0, metaResources.size()); + + query = "SELECT c3, c4, c5 from " + tableName; + resources = executeQueryAndGetResources(query); + verifyIDFieldsCount(resources, 0); // no ID fields + metaResources = resources.stream().filter(resource -> resource.getType().equals("meta")).collect(Collectors.toList()); + Assert.assertEquals(0, metaResources.size()); + } + + private static void verifyIDFieldsCount(List resources, long expectedCount) { + long fieldsWithIDCount = resources.stream() + .filter(resource -> resource.getTable() != null) + .flatMap(resource -> resource.getTable().getFields().stream()) + .filter(field -> field.id != null) + .count(); + + Assert.assertEquals(expectedCount, fieldsWithIDCount); + } + + private void createCustomTable(String tableName) throws Exception { + clearSchemaPerms(); + + TapPermissions tapPermissions = new TapPermissions(null, true, null, null); + super.setPerms(schemaOwner, testSchemaName, tapPermissions, 200); + + doDelete(schemaOwner, tableName, false); + TableDesc orig = prepareTable(tableName); + uploadTable(tableName, orig); + + log.info("Table " + tableName + " created successfully."); + } + + private void createServiceDescriptorTemplates() throws IOException, ResourceNotFoundException { + uploadServiceDescriptorTemplate("service-descriptor-template-1.xml", "testServiceDescriptorTemplate1"); + uploadServiceDescriptorTemplate("service-descriptor-template-2.xml", "testServiceDescriptorTemplate2"); + + log.info("Service descriptor templates created successfully."); + } + + private void uploadServiceDescriptorTemplate(String sdFileName, String sdName) throws IOException, ResourceNotFoundException { + File testFile = FileUtil.getFileFromResource(sdFileName, YoucatTableWriterTest.class); + String template = Files.readString(testFile.toPath()); + + JSONObject json = new JSONObject(); + json.put("name", sdName); + json.put("template", template); + + FileContent content = new FileContent(json.toString().getBytes(StandardCharsets.UTF_8), "application/json"); + /*HttpPost httpPost = new HttpPost(templatesURL, content, false); // TODO: use the actual URL for templates when available + httpPost.run(); + + Assert.assertNull(httpPost.getThrowable()); + Assert.assertEquals(200, httpPost.getResponseCode());*/ + } + + private void uploadTable(String tableName, TableDesc orig) throws Exception { + URL tableURL = new URL(certTablesURL.toExternalForm() + "/" + tableName); + TableWriter w = new TableWriter(); + StringWriter sw = new StringWriter(); + w.write(orig, sw); + log.info("VOSI-table description:\n" + sw.toString()); + + OutputStreamWrapper src = new OutputStreamWrapper() { + @Override + public void write(OutputStream out) throws IOException { + TableWriter w = new TableWriter(); + w.write(orig, new OutputStreamWriter(out)); + } + }; + HttpUpload put = new HttpUpload(src, tableURL); + put.setContentType(TablesInputHandler.VOSI_TABLE_TYPE); + log.info("doCreateTable: " + tableURL); + Subject.doAs(schemaOwner, new RunnableAction(put)); + log.info("doCreateTable: " + put.getResponseCode()); + if (put.getThrowable() != null && put.getThrowable() instanceof Exception) { + throw (Exception) put.getThrowable(); + } + Assert.assertEquals("response code", 200, put.getResponseCode()); + } + + private TableDesc prepareTable(String tableName) { + final TableDesc orig = new TableDesc(testSchemaName, tableName); + orig.description = "created by intTest"; + orig.tableType = TableDesc.TableType.TABLE; + orig.tableIndex = 1; + + ColumnDesc c0 = new ColumnDesc(tableName, "c0", TapDataType.STRING); + c0.columnID = "testServiceDescriptorID1"; + orig.getColumnDescs().add(c0); + + ColumnDesc c1 = new ColumnDesc(tableName, "c1", TapDataType.SHORT); + c1.columnID = "testServiceDescriptorID2"; + orig.getColumnDescs().add(c1); + + ColumnDesc c2 = new ColumnDesc(tableName, "c2", TapDataType.INTEGER); + c2.columnID = "testServiceDescriptorID3"; + orig.getColumnDescs().add(c2); + + orig.getColumnDescs().add(new ColumnDesc(tableName, "c3", TapDataType.INTEGER)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "c4", TapDataType.INTEGER)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "c5", TapDataType.INTEGER)); + return orig; + } + + private List executeQueryAndGetResources(String query) throws Exception { + String result = doQuery(query); + VOTableReader r = new VOTableReader(); + VOTableDocument doc = r.read(result); + + return doc.getResources(); + } + + private String doQuery(String query) throws Exception { + Map params = new TreeMap(); + params.put("LANG", "ADQL"); + params.put("QUERY", query); + String result = Subject.doAs(schemaOwner, new AuthQueryTest.SyncQueryAction(anonQueryURL, params)); + Assert.assertNotNull(result); + return result; + } +} \ No newline at end of file diff --git a/youcat/src/intTest/resources/service-descriptor-template-1.xml b/youcat/src/intTest/resources/service-descriptor-template-1.xml new file mode 100644 index 00000000..c97e4872 --- /dev/null +++ b/youcat/src/intTest/resources/service-descriptor-template-1.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/youcat/src/intTest/resources/service-descriptor-template-2.xml b/youcat/src/intTest/resources/service-descriptor-template-2.xml new file mode 100644 index 00000000..fc4ee07b --- /dev/null +++ b/youcat/src/intTest/resources/service-descriptor-template-2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/youcat/src/main/java/org/opencadc/youcat/YoucatTableWriter.java b/youcat/src/main/java/org/opencadc/youcat/YoucatTableWriter.java new file mode 100644 index 00000000..5dae962e --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/YoucatTableWriter.java @@ -0,0 +1,37 @@ +package org.opencadc.youcat; + +import ca.nrc.cadc.dali.tables.votable.VOTableDocument; +import ca.nrc.cadc.dali.tables.votable.VOTableResource; +import ca.nrc.cadc.tap.DefaultTableWriter; +import ca.nrc.cadc.tap.PluginFactory; + +import java.io.IOException; +import java.util.List; + +import org.apache.log4j.Logger; +import org.opencadc.datalink.ServiceDescriptorTemplate; + +public class YoucatTableWriter extends DefaultTableWriter { + + private static final Logger log = Logger.getLogger(YoucatTableWriter.class); + + @Override + protected void addMetaResources(VOTableDocument votableDocument, List fieldIDs) throws IOException { + super.addMetaResources(votableDocument, fieldIDs); + + // Use TemplateDAO to get descriptors + TemplateDAO templateDAO = new TemplateDAO(new PluginFactory().getTapSchemaDAO()); + List templates = templateDAO.list(fieldIDs); + + for (ServiceDescriptorTemplate template : templates) { + if (fieldIDs.containsAll(template.getIdentifiers())) { + VOTableResource resource = template.getResource(); + if (resource == null) { + log.warn("No resource found for template: " + template.getName()); + } + votableDocument.getResources().add(resource); + populateAccessURLParam(resource); + } + } + } +} diff --git a/youcat/src/main/resources/PluginFactory.properties b/youcat/src/main/resources/PluginFactory.properties index 05117c3b..5c252f04 100644 --- a/youcat/src/main/resources/PluginFactory.properties +++ b/youcat/src/main/resources/PluginFactory.properties @@ -14,7 +14,7 @@ ca.nrc.cadc.tap.db.DatabaseDataType=ca.nrc.cadc.tap.pg.PostgresDataTypeMapper ca.nrc.cadc.tap.UploadManager = org.opencadc.youcat.tap.UploadManagerImpl -#ca.nrc.cadc.tap.TableWriter = ca.nrc.cadc.tap.DefaultTableWriter +ca.nrc.cadc.tap.TableWriter = org.opencadc.youcat.YoucatTableWriter ca.nrc.cadc.tap.writer.format.FormatFactory = org.opencadc.youcat.tap.FormatFactoryImpl