Changeset 161 for trunk/corelib/src/org/bridgedb/webservice/biomart/BiomartStub.java

Timestamp:

08/19/09 05:53:32 (3 years ago)

Author:

jgao

Message:

IDMapperBiomart refactor to clean the interface

Files:

• trunk/corelib/src/org/bridgedb/webservice/biomart/BiomartStub.java (modified) (2 diffs)

trunk/corelib/src/org/bridgedb/webservice/biomart/BiomartStub.java

@@ -19 +19 @@
 
 import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
-
-import java.net.URL;
-import java.net.URLConnection;
 
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
+import java.util.Set;
 import java.util.Vector;
 
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
-import org.w3c.dom.Document;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.NodeList;
+import org.bridgedb.DataSource;
+import org.bridgedb.IDMapperException;
+import org.bridgedb.Xref;
+import org.bridgedb.file.IDMappingReaderFromDelimitedReader;
+import org.bridgedb.webservice.biomart.util.Attribute;
+import org.bridgedb.webservice.biomart.util.BiomartClient;
+import org.bridgedb.webservice.biomart.util.Database;
+import org.bridgedb.webservice.biomart.util.Dataset;
+import org.bridgedb.webservice.biomart.util.Filter;
+import org.bridgedb.webservice.biomart.util.XMLQueryBuilder;
 
 import org.xml.sax.SAXException;
 
 /**
- * BioMart service class, adapted from BioMart client in Cytoscape.
+ * Cache for SynergizerClient
+ * @author gjj
  */
-public final class BiomartStub {
-    public static final String defaultBaseURL = "http://www.biomart.org/biomart/martservice";
-    
-    private final String baseURL;
-    private static final String RESOURCE = "/org/bridgedb/webservice/biomart/filterconversion.txt";
-
-    //private Map<String, Map<String, String>> databases = null;
-    private Map<String, Database> databases = null;
-
-    private Map<String, Dataset> datasets = new HashMap();
-    private Map<String, Vector<Dataset>> mapDbDss = new HashMap();
-    private Map<String, Map<String, Filter>> mapDsFilters = new HashMap();
-    private Map<String, Map<String, Attribute>> mapDsAttrs = new HashMap();
-
-    private Map<String, Map<String, String>> filterConversionMap;
-
-    private static final int BUFFER_SIZE = 81920;
-
-    // one instance per base url
+public class BiomartStub {
+    public static final String defaultBaseURL
+            = BiomartClient.defaultBaseURL;
+
+    // cache data
+//    private Map<String,Map<String,Map<String,Set<String>>>>
+//            mapAuthSpeciesDomainRange = null;
+
+    // one instance per url
     private static Map<String, BiomartStub> instances = new HashMap();
 
     /**
-     * Get a BioMartStub with the default base URL.
-     * @return BiomartStub
-     * @throws IOException if failed to read local resource
+     *
+     * @return SynergizerStub with the default server url
+     * @throws IOException if failed to connect
      */
     public static BiomartStub getInstance() throws IOException {
@@ -76 +68 @@
 
     /**
-     * Get a BioMartStub with the base URL.
-     * @param baseURL base URL of BioMart
-     * @return BioMartStub
-     * @throws IOException if failed to read local resource
-     */
-    public static BiomartStub getInstance(String baseURL) throws IOException {
-        if (baseURL==null) {
+     *
+     * @param baseUrl server url
+     * @return SynergizerStub from the server
+     * @throws IOException if failed to connect
+     */
+    public static BiomartStub getInstance(String baseUrl) throws IOException {
+        if (baseUrl==null) {
             throw new IllegalArgumentException("base url cannot be null");
         }
 
-        BiomartStub instance = instances.get(baseURL);
+        BiomartStub instance = instances.get(baseUrl);
         if (instance==null) {
-            instance = new BiomartStub(baseURL);
-            instances.put(baseURL, instance);
+            instance = new BiomartStub(baseUrl);
+            instances.put(baseUrl, instance);
         }
 
         return instance;
-   }
-
-
-    /**
-     * Creates a new BiomartStub object from given URL.
-     *
-     * @param baseURL  DOCUMENT ME!
-     * @throws IOException if failed to read local resource
-     */
-    private BiomartStub(String baseURL) throws IOException {
-        this.baseURL = baseURL + "?";
-        loadConversionFile();
-    }
-
-    /**
-     * Conversion map from filter to attribute.
-     * @throws IOException if failed to read local resource
-     */
-    private void loadConversionFile() throws IOException {
-        filterConversionMap = new HashMap<String, Map<String, String>>();
-
-        InputStreamReader inFile = new InputStreamReader(this.getClass().getResource(RESOURCE).openStream());
-
-        BufferedReader inBuffer = new BufferedReader(inFile);
-
-        String line;
-        String trimed;
-        String oldName = null;
-        Map<String, String> oneEntry = new HashMap<String, String>();
-
-        String[] dbparts;
-
-        while ((line = inBuffer.readLine()) != null) {
-            trimed = line.trim();
-            dbparts = trimed.split("\\t");
-
-            if (dbparts[0].equals(oldName) == false) {
-                oneEntry = new HashMap<String, String>();
-                oldName = dbparts[0];
-                filterConversionMap.put(oldName, oneEntry);
+    }
+
+    private BiomartClient client;
+
+    /**
+     *
+     * @param baseUrl server url.
+     * @throws IOException if failed to connect.
+     */
+    private BiomartStub(String baseUrl) throws IOException {
+        client = new BiomartClient(baseUrl);
+    }
+
+    /**
+     *
+     * @return available marts
+     * @throws IDMapperException if failed
+     */
+    public Set<String> availableMarts() throws IDMapperException {
+        Map<String, Database> marts;
+        try {
+            marts = client.getRegistry();
+        } catch (IOException e) {
+            throw new IDMapperException(e);
+        } catch (ParserConfigurationException e) {
+            throw new IDMapperException(e);
+        } catch (SAXException e) {
+            throw new IDMapperException(e);
+        }
+
+        Set<String> visibleMarts = new HashSet();
+        for (Database db : marts.values()) {
+            if (db.visible()) {
+                visibleMarts.add(db.getName());
             }
-
-            oneEntry.put(dbparts[1], dbparts[2]);
-        }
-
-        inFile.close();
-        inBuffer.close();
-    }
-
-    /**
-     * Convert filter to attribute according to the conversion file.
-     * @param dsName dataset name.
-     * @param dbName database name
-     * @param filterID filter ID
-     * @return converted attribute
-     */
-    public Attribute filterToAttributeName(String dsName, String dbName,
-                String filterID) {
-        if (filterConversionMap.get(dbName) == null) {
+        }
+
+        return visibleMarts;
+    }
+
+    /**
+     *
+     * @param mart mart name
+     * @return mart display name or null if not exist
+     * @throws IDMapperException if failed to connect
+     */
+    public String martDisplayName(String mart) {
+        if (mart==null) {
             return null;
-        } else {
-            String attrName = filterConversionMap.get(dbName).get(filterID);
-            return this.getAttribute(dsName, attrName);
-        }
-    }
-
-    /**
-     *  Get the registry information from the base URL.
-     *
-     * @return  Map of registry information.  Key value is "name" field.
-     * @throws ParserConfigurationException if failed new document builder
-     * @throws SAXException if failed to parse registry
-     * @throws IOException if failed to read from URL
-     */
-    public Map<String, Database> getRegistry()
-            throws IOException, ParserConfigurationException, SAXException {
-        // If already loaded, just return it.
-        if (databases != null)
-            return databases;
-
-        // Initialize database map.
-        databases = new HashMap<String, Database>();
-
-        // Prepare URL for the registry status
-        final String reg = "type=registry";
-        final URL targetURL = new URL(baseURL + reg);
-
-        // Get the result as XML document.
-        final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
-        final DocumentBuilder builder = factory.newDocumentBuilder();
-
-        InputStream is = getInputStream(targetURL);
-
-        final Document registry = builder.parse(is);
-
-        // Extract each datasource
-        NodeList locations = registry.getElementsByTagName("MartURLLocation");
-        int locSize = locations.getLength();
-        NamedNodeMap attrList;
-        int attrLen;
-        String dbID;
-
-        for (int i = 0; i < locSize; i++) {
-            attrList = locations.item(i).getAttributes();
-            attrLen = attrList.getLength();
-
-            // First, get the key value
-            dbID = attrList.getNamedItem("name").getNodeValue();
-
-            Map<String, String> entry = new HashMap<String, String>();
-
-            for (int j = 0; j < attrLen; j++) {
-                entry.put(attrList.item(j).getNodeName(), attrList.item(j).getNodeValue());
-            }
-
-            databases.put(dbID, new Database(dbID,entry));
-        }
-
-        is.close();
-        is = null;
-
-        return databases;
-    }
-
-    /**
-     * Get available datasets of a mart/database.
-     * @param martName mart name
-     * @return {@link Vector} of available datasets
-     * @throws IOException if failed to read
-     */
-    public Vector<Dataset> getAvailableDatasets(final String martName)
-            throws IOException {
-        Vector<Dataset> result = mapDbDss.get(martName);
-        if (result!=null) {
-            return result;
-        }
-
-        try {
-            getRegistry();
-        } catch (ParserConfigurationException e) {
-            e.printStackTrace();
-        } catch (SAXException e) {
-            e.printStackTrace();
-        }
-
-        //final Map<String, String> datasources = new HashMap<String, String>();
-        result = new Vector();
-
-        Database database = databases.get(martName);
-
-        if (database==null) {
+        }
+
+        Database db = client.getMart(mart);
+        return db==null?null:db.displayName();
+    }
+
+    /**
+     *
+     * @param authority mart name
+     * @return available datasets from this mart
+     * @throws IDMapperException if failed
+     */
+    public Set<String> availableDatasets(String mart)
+            throws IDMapperException {
+        if (mart==null) {
+            return new HashSet(0);
+        }
+
+        if (!availableMarts().contains(mart)) {
+            return new HashSet(0);
+        }
+
+        Map<String,Dataset> datasets;
+        try {
+            datasets = client.getAvailableDatasets(mart);
+        } catch (IOException e) {
+            throw new IDMapperException(e);
+        }
+
+        return datasets.keySet();
+    }
+
+     /**
+     *
+     * @param dataset dataset name
+     * @return dataset display name or null if not exist
+     * @throws IDMapperException if failed to connect
+     */
+    public String datasetDisplayName(String dataset) {
+        if (dataset==null) {
             return null;
         }
 
-        Map<String, String> detail = database.getParam();
-
-        String urlStr = "http://" + detail.get("host") + ":" + detail.get("port")
-                        + detail.get("path") + "?type=datasets&mart=" + detail.get("name");
-        //System.out.println("DB name = " + martName + ", Target URL = " + urlStr + "\n");
-
-        URL url = new URL(urlStr);
-        InputStream is = getInputStream(url);
-
-        BufferedReader reader = new BufferedReader(new InputStreamReader(is));
-        String s;
-
-        String[] parts;
-
-        while ((s = reader.readLine()) != null) {
-            parts = s.split("\\t");
-
-            if ((parts.length > 4) && parts[3].equals("1")) {
-                Dataset dataset = new Dataset(parts[1], parts[2], database);
-                result.add(dataset);
-                //datasourceMap.put(parts[1], martName);
-                datasets.put(parts[1], dataset);
-            }
-        }
-
-        is.close();
-        reader.close();
-        reader = null;
-        is = null;
-
-        mapDbDss.put(martName, result);
-
-        return result;
-    }
-
-    /**
-     * Get filters for a dataset.
-     * @param datasetName name of data set
-     * @return filters
-     * @throws IOException if failed to read
-     */
-    public Map<String, Filter> getFilters(String datasetName)
-        throws IOException {
-        if (datasetName==null) {
-            throw new IllegalArgumentException("Dataset name cannot be null");
-        }
-
-        if (mapDsFilters.get(datasetName)!=null) {
-            return mapDsFilters.get(datasetName);
-        }
-
-        Map<String, Filter> filters = new HashMap();
-
-        Dataset dataset = getDataset(datasetName);
-        if (dataset==null) {
-            return filters;
-        }
-
-        Database database = dataset.getDatabase();
-
-        Map<String, String> detail = database.getParam();
-
-        String urlStr = "http://" 
-                        + detail.get("host") + ":"
-                        + detail.get("port")
-                        + detail.get("path")
-                        + "?virtualschema="
-                        + detail.get("serverVirtualSchema")
-                        + "&type=filters&dataset="
-                        + datasetName;
-
-        //System.out.println("Dataset name = " + datasetName + ", Target URL = "
-        //                      + urlStr + "\n");
-        URL url = new URL(urlStr);
-        InputStream is = getInputStream(url);
-
-        BufferedReader reader = new BufferedReader(new InputStreamReader(is));
-        String s;
-
-        String[] parts;
-
-        while ((s = reader.readLine()) != null) {
-            parts = s.split("\\t");
-
-            if ((parts.length > 1)) {
-                if ((parts[1].contains("ID(s)")
-                            || parts[1].contains("Accession(s)")
-                            || parts[1].contains("IDs"))
-                         && (parts[0].startsWith("with_") == false)
-                         && (parts[0].endsWith("-2") == false)
-                         || parts.length>6
-                         && parts[5].equals("id_list")) {
-                    //filters.put(parts[1], parts[0]);
-                    filters.put(parts[0], new Filter(parts[0], parts[1]));
-                    // System.out.println("### Filter Entry = " + parts[1] + " = " + parts[0]);
+        Dataset ds = client.getDataset(dataset);
+        return ds==null?null:ds.displayName();
+    }
+
+    /**
+     *
+     * @param mart mart name
+     * @param dataset dataset name
+     * @return available filters / source id types of the dataset from this
+     *         mart
+     * @throws IDMapperException if failed
+     */
+    public Set<String> availableFilters(final String mart,
+            final String dataset) throws IDMapperException {
+        if (mart==null || dataset==null) {
+            return new HashSet(0);
+        }
+
+        if (!availableDatasets(mart).contains(dataset)) {
+            return new HashSet(0);
+        }
+
+        Map<String,Filter> filters;
+        try {
+            filters = client.getFilters(dataset);
+        } catch (IOException e) {
+            throw new IDMapperException(e);
+        }
+
+        return filters.keySet();
+    }
+
+    /**
+     *
+     * @param mart mart name
+     * @param dataset dataset name
+     * @return attribute names / target id types of the dataset from this
+     *         mart
+     * @throws IDMapperException if failed.
+     */
+    public Set<String> availableAttributes(final String mart,
+            final String dataset, boolean idOnly) throws IDMapperException {
+
+        if (mart==null || dataset==null) {
+            return new HashSet(0);
+        }
+
+        if (!availableDatasets(mart).contains(dataset)) {
+            return new HashSet(0);
+        }
+
+        Map<String,Attribute> attributes;
+        try {
+            attributes = client.getAttributes(dataset);
+        } catch (IOException e) {
+            throw new IDMapperException(e);
+        }
+
+        Set<String> result;
+        if (idOnly) {
+            result = new HashSet();
+            for (String name : attributes.keySet()) {
+                String displayName = client.getAttribute(dataset, name).getDisplayName();
+                if (displayName.endsWith("ID")
+                        || displayName.endsWith("Accession")
+                        || name.endsWith("id")
+                        || name.endsWith("accession")) {
+                    result.add(name);
                 }
             }
-        }
-
-        is.close();
-        reader.close();
-        reader = null;
-        is = null;
-
-        mapDsFilters.put(datasetName, filters);
-
-        return filters;
-    }
-
-    /**
-     * Get attributes.
-     * @param datasetName dataset name
-     * @return Map of attribute name to attributes
-     * @throws IOException if failed to read
-     */
-    public Map<String, Attribute> getAttributes(String datasetName) throws IOException {
-        if (datasetName==null) {
-            throw new java.lang.IllegalArgumentException("Dataset name cannot be null");
-        }
-
-        if (mapDsAttrs.get(datasetName)!=null) {
-            return mapDsAttrs.get(datasetName);
-        }
-
-        Map<String, Attribute> attributes = new HashMap<String, Attribute>();
-
-        Dataset dataset = getDataset(datasetName);
-        if (dataset==null) {
-            return attributes;
-        }
-
-        Database database = dataset.getDatabase();
-
-        Map<String, String> detail = database.getParam();
-
-        String urlStr = "http://" + detail.get("host") + ":" + detail.get("port")
-                        + detail.get("path") + "?virtualschema="
-                        + detail.get("serverVirtualSchema") + "&type=attributes&dataset="
-                        + datasetName;
-
-        //System.out.println("Dataset name = " + datasetName + ", Target URL = " + urlStr + "\n");
-        URL url = new URL(urlStr);
-        InputStream is = getInputStream(url);
-
-        BufferedReader reader = new BufferedReader(new InputStreamReader(is));
-        String s;
-
-        String displayName;
-
-        String[] parts;
-
-        while ((s = reader.readLine()) != null) {
-            parts = s.split("\\t");
-
-            if (parts.length == 0)
-                continue;
-
-            if (parts.length == 4) {
-                displayName = parts[3] + ": " + parts[1];
-            } else if (parts.length > 1) {
-                displayName = parts[1];
-            } else {
-                displayName = "";
+        } else {
+            result = new HashSet(attributes.keySet());
+        }
+
+        return result;
+    }
+
+    /**
+     * 
+     * @param mart mart name
+     * @param dataset dataset name
+     * @param filter filter name / source id type
+     * @param attributes attribute names / target id types
+     * @param ids source ids to be translated
+     * @return map from source id to target ids
+     *         key: source id
+     *         value: corresponding target ids
+     * @throws IDMapperException
+     */
+    public Map<String,Set<String>[]> translate(final String mart,
+            final String dataset, final String filter,
+            final String[] attributes, final Set<String> ids)
+            throws IDMapperException {
+        int nAttr = attributes.length;
+        Attribute[] attrs = new Attribute[nAttr+1];
+
+        // prepare attributes
+        int iattr = 0;
+        for (String attr : attributes) {
+            attrs[iattr++] = client.getAttribute(dataset, attr);
+        }
+        attrs[nAttr] = client.filterToAttribute(dataset, filter);
+
+        // prepare filters
+        StringBuilder sb = new StringBuilder();
+        for (String str : ids) {
+            sb.append(str);
+            sb.append(",");
+        }
+
+        int len = sb.length();
+        if (len>0) {
+            sb.deleteCharAt(len-1);
+        }
+
+        Map<String, String> queryFilter = new HashMap(1);
+        queryFilter.put(filter, sb.toString());
+
+        // build query string
+        String query = XMLQueryBuilder.getQueryString(dataset, attrs, queryFilter);
+
+        // query
+        BufferedReader bfr = null;
+        try {
+            bfr = client.sendQuery(query);
+            if (!bfr.ready())
+                throw new IDMapperException("Query failed");
+        } catch (IOException e) {
+            throw new IDMapperException(e);
+        }
+
+        if (bfr==null) {
+            return new HashMap(0);
+        }
+
+        // read id mapping
+        Map<String,Set<String>[]> result = new HashMap();
+        try {
+            bfr.readLine();
+            String line;
+            while ((line = bfr.readLine())!=null) {
+                String[] strs = line.split("\t");
+                if (strs.length!=nAttr+1)
+                    continue; // because the last one is the src id
+                String src = strs[nAttr];
+                Set<String>[] tgt = result.get(src);
+                if (tgt==null) {
+                    tgt = new Set[nAttr];
+                    for (int i=0; i<nAttr; i++) {
+                        tgt[i] = new HashSet();
+                    }
+                    result.put(src, tgt);
+                }
+
+                for (int i=0; i<nAttr; i++) {
+                    String str = strs[i];
+                    if (str.length()>0) {
+                        tgt[i].add(str);
+                    }
+                }
+
             }
-
-            attributes.put(parts[0], new Attribute(parts[0],displayName));
-        }
-
-        is.close();
-        reader.close();
-        reader = null;
-        is = null;
-
-        this.mapDsAttrs.put(datasetName, attributes);
-
-        return attributes;
-    }
-
-    /**
-     * Get filter.
-     * @param datasetName dataset name
-     * @param filterName filter name
-     * @return filter
-     */
-    public Filter getFilter(String datasetName, String filterName) {
-        if (datasetName==null || filterName==null) {
-            throw new java.lang.IllegalArgumentException("datasetName and filterName cannot be null");
-        }
-
-        Map<String, Filter> map = this.mapDsFilters.get(datasetName);
-        if (map==null) {
-            return null;
-        }
-
-        return map.get(filterName);
-    }
-
-    /**
-     * get Attribute.
-     * @param datasetName dataset name
-     * @param attrName attribute name
-     * @return attribute
-     */
-    public Attribute getAttribute(String datasetName, String attrName) {
-        if (datasetName==null || attrName==null) {
-            throw new java.lang.IllegalArgumentException("datasetName and attrName cannot be null");
-        }
-
-        Map<String, Attribute> map = this.mapDsAttrs.get(datasetName);
-        if (map==null) {
-            return null;
-        }
-
-        return map.get(attrName);
-    }
-
-    /**
-     * Send the XML query to Biomart, and get the result as table.
-     * @param xmlQuery query xml
-     * @return result {@link BufferedReader}
-     * @throws IOException if failed to read
-     */
-    public BufferedReader sendQuery(String xmlQuery) throws IOException {
-
-        //System.out.println("=======Query = " + xmlQuery);
-
-        URL url = new URL(baseURL);
-        URLConnection uc = url.openConnection();
-        uc.setDoOutput(true);
-        uc.setRequestProperty("User-Agent", "Java URLConnection");
-
-        OutputStream os = uc.getOutputStream();
-
-        final String postStr = "query=" + xmlQuery;
-        PrintStream ps = new PrintStream(os);
-
-        // Post the data
-        ps.print(postStr);
-        os.close();
-        ps.close();
-        ps = null;
-        os = null;
-
-        return new BufferedReader(new InputStreamReader(uc.getInputStream()), BUFFER_SIZE);
-    }
-
-    /**
-     * get Database/mart.
-     * @param dbname database name
-     * @return database
-     */
-    public Database getDatabase(final String dbname) {
-        return databases.get(dbname);
-    }
-
-    /**
-     * get Dataset.
-     * @param dsname dataset name
-     * @return dataset
-     */
-    public Dataset getDataset(final String dsname) {
-        return datasets.get(dsname);
-    }
-    
-    private static final int msConnectionTimeout = 2000;
-    //TODO: test when IOException is throwed
-    protected static InputStream getInputStream(URL source) throws IOException {
-        InputStream stream = null;
-        int expCount = 0;
-        int timeOut = msConnectionTimeout;
-        while (true) { // multiple chances
-            try {
-                URLConnection uc = source.openConnection();
-                uc.setUseCaches(false); // don't use a cached page
-                uc.setConnectTimeout(timeOut); // set timeout for connection
-                stream = uc.getInputStream();
-                break;
-            } catch (IOException e) {
-                if (expCount++==4) {
-                    throw(e);
-                } else {
-                    timeOut *= 2;
-                }
-            }
-        }
-
-        return stream;
+        } catch (IOException e) {
+            throw new IDMapperException(e);
+        }
+
+        return result;
     }
 }