/**
* '$RCSfile$'
* Purpose: A Class that implements administrative methods
* Copyright: 2010 Regents of the University of California and the
* National Center for Ecological Analysis and Synthesis
* Authors: Michael Daigle
*
* '$Author: berkley $'
* '$Date: 2010-06-08 12:34:30 -0700 (Tue, 08 Jun 2010) $'
* '$Revision: 5374 $'
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
package edu.ucsb.nceas.metacat.util;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.math.BigInteger;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import org.apache.commons.io.IOUtils;
import org.dataone.client.v2.itk.D1Client;
import org.dataone.client.v2.MNode;
import org.dataone.client.v2.formats.ObjectFormatCache;
import org.dataone.client.auth.CertificateManager;
import org.dataone.ore.ResourceMapFactory;
import org.dataone.service.exceptions.NotFound;
import org.dataone.service.types.v1.AccessPolicy;
import org.dataone.service.types.v1.AccessRule;
import org.dataone.service.types.v1.Checksum;
import org.dataone.service.types.v1.Identifier;
import org.dataone.service.types.v1.NodeReference;
import org.dataone.service.types.v1.ObjectFormat;
import org.dataone.service.types.v1.Permission;
import org.dataone.service.types.v1.Session;
import org.dataone.service.types.v1.Subject;
import org.dataone.service.types.v2.SystemMetadata;
import org.dataone.service.types.v1.util.ChecksumUtil;
import org.dataone.service.util.Constants;
import org.dspace.foresite.ResourceMap;
import org.ecoinformatics.datamanager.DataManager;
import org.ecoinformatics.datamanager.database.DatabaseConnectionPoolInterface;
import org.ecoinformatics.datamanager.parser.DataPackage;
import edu.ucsb.nceas.metacat.MetaCatServlet;
import edu.ucsb.nceas.metacat.dataquery.MetacatDatabaseConnectionPoolFactory;
import edu.ucsb.nceas.metacat.properties.PropertyService;
/**
* @author berkley
* A class to populate a metacat instance based on documents returned from a query
*/
public class MetacatPopulator
{
private String sourceUrl = null;
private String destinationUrl = null;
private String query = null;
private String username = null;
private String password = null;
private Session session = null;
private String subjectDN = null;
/**
* create a new MetacatPopulator with given source and destination urls.
* These should be
* of the form "http://<url>/<metacat_instance>"
* If username and/or password is null, the query will be run as public
* @param sourceUrl
* @param destUrl
* @param query
* @param username
* @param password
*/
public MetacatPopulator(String sourceUrl, String destUrl, String query, String username, String password)
{
this.sourceUrl = sourceUrl;
this.query = query;
this.username = username;
this.password = password;
this.destinationUrl = destUrl;
// TODO: use specific certificate?
this.session = null; //new Session();
this.subjectDN = CertificateManager.getInstance().getSubjectDN(CertificateManager.getInstance().loadCertificate());
}
/**
* populate from the source
*/
public void populate()
throws Exception
{
//String sourceSessionid = login();
//do a query
String params = "returndoctype=eml://ecoinformatics.org/eml-2.1.0&" +
"returndoctype=eml://ecoinformatics.org/eml-2.0.1&" +
"returndoctype=eml://ecoinformatics.org/eml-2.0.0&";
params += "action=query&";
params += "qformat=xml&";
params += "anyfield=" + query;
printHeader("Searching source");
System.out.println("searching '" + sourceUrl + "' for '" + query + "'");
InputStream is = getResponse(sourceUrl, "/metacat", params, "POST");
String response = IOUtils.toString(is, MetaCatServlet.DEFAULT_ENCODING);
//System.out.println("response: " + response);
Vector<Document> docs = parseResponse(response);
printHeader("Parsing source results");
System.out.println("creating MN with url: " + destinationUrl + "/");
MNode mn = D1Client.getMN(destinationUrl + "/");
printHeader("Processing " + docs.size() + " results.");
for (int i=0; i<docs.size(); i++) {
// for generating the ORE map
Map<Identifier, List<Identifier>> idMap = new HashMap<Identifier, List<Identifier>>();
List<Identifier> dataIds = new ArrayList<Identifier>();
//for each document in the query
Document doc = docs.get(i);
String docid = doc.docid;
//get the doc from source
printHeader("Getting document " + doc.docid + " from source " + sourceUrl);
params = "action=read&qformat=xml&docid=" + docid;
is = getResponse(sourceUrl, "/metacat", params, "POST");
String doctext = IOUtils.toString(is, MetaCatServlet.DEFAULT_ENCODING);
System.out.println("doctext: " + doctext);
is = IOUtils.toInputStream(doctext, MetaCatServlet.DEFAULT_ENCODING);
//parse the document
DatabaseConnectionPoolInterface connectionPool = MetacatDatabaseConnectionPoolFactory.getDatabaseConnectionPoolInterface();
DataManager dataManager = DataManager.getInstance(connectionPool, connectionPool.getDBAdapterName());
DataPackage dataPackage = dataManager.parseMetadata(is);
if (dataPackage == null) {
continue;
}
//go through the DistributionMetadata and download any described data
is = IOUtils.toInputStream(doctext, MetaCatServlet.DEFAULT_ENCODING);
doc.doctext = doctext;
printHeader("creating document on destination " + destinationUrl);
SystemMetadata sysmeta = generateSystemMetadata(doc);
// iterate through the data objects
if (dataPackage.getEntityList() != null) {
for (int j=0; j < dataPackage.getEntityList().length; j++) {
String dataDocUrl = dataPackage.getEntityList()[j].getURL();
String dataDocMimeType = dataPackage.getEntityList()[j].getDataFormat();
if (dataDocMimeType == null) {
dataDocMimeType =
ObjectFormatCache.getInstance().getFormat("application/octet-stream").getFormatId().getValue();
}
String dataDocLocalId = "";
if (dataDocUrl.trim().startsWith("ecogrid://knb/")) { //we only handle ecogrid urls right now
dataDocLocalId = dataDocUrl.substring(dataDocUrl.indexOf("ecogrid://knb/") +
"ecogrid://knb/".length(), dataDocUrl.length());
//get the file
params = "action=read&qformat=xml&docid=" + dataDocLocalId;
InputStream dataDocIs = getResponse(sourceUrl, "/metacat", params, "POST");
String dataDocText = IOUtils.toString(dataDocIs, MetaCatServlet.DEFAULT_ENCODING);
//set the id
Identifier did = new Identifier();
did.setValue(dataDocLocalId);
// add the data identifier for ORE map
dataIds.add(did);
//create sysmeta for the data doc
SystemMetadata dataDocSysMeta = generateSystemMetadata(doc);
//overwrite the bogus values from the last call
dataDocSysMeta.setIdentifier(did);
ObjectFormat format = null;
try {
format = ObjectFormatCache.getInstance().getFormat(dataDocMimeType);
dataDocSysMeta.setFormatId(format.getFormatId());
} catch (NotFound e) {
System.out.println(e.getMessage());
}
dataDocIs = IOUtils.toInputStream(dataDocText, MetaCatServlet.DEFAULT_ENCODING);
String algorithm = PropertyService.getProperty("dataone.checksumAlgorithm.default");
Checksum checksum = ChecksumUtil.checksum(dataDocIs, algorithm);
dataDocSysMeta.setChecksum(checksum);
String sizeStr =
Long.toString(dataDocText.getBytes(MetaCatServlet.DEFAULT_ENCODING).length);
dataDocSysMeta.setSize(new BigInteger(sizeStr));
boolean error = false;
//create the data doc on d1
try {
mn.create(session, dataDocSysMeta.getIdentifier(), IOUtils.toInputStream(dataDocText, MetaCatServlet.DEFAULT_ENCODING), dataDocSysMeta);
}
catch(Exception e) {
error = true;
System.out.println("ERROR: Could not create data document with id " +
dataDocSysMeta.getIdentifier().getValue() + " : " + e.getMessage());
}
finally {
if (error) {
printHeader("Insertion of document " + dataDocSysMeta.getIdentifier().getValue() +
"FAILED.");
}
else {
printHeader("Done inserting document " + dataDocSysMeta.getIdentifier().getValue() +
" which is described by " + sysmeta.getIdentifier().getValue());
}
}
}
else {
System.out.println("WARNING: Could not process describes url " +
dataDocUrl + " for document " + doc.docid +
". Only ecogrid://knb/ urls are currently supported.");
}
}
}
try {
Identifier id =
mn.create(session, sysmeta.getIdentifier(), IOUtils.toInputStream(doc.doctext, MetaCatServlet.DEFAULT_ENCODING), sysmeta);
System.out.println("Success inserting document " + id.getValue());
// no need for an ORE map if there's no data
if (!dataIds.isEmpty()) {
// generate the ORE map for this datapackage
Identifier resourceMapId = new Identifier();
resourceMapId.setValue("resourceMap_" + sysmeta.getIdentifier().getValue());
idMap.put(sysmeta.getIdentifier(), dataIds);
ResourceMap rm = ResourceMapFactory.getInstance().createResourceMap(resourceMapId, idMap);
String resourceMapXML = ResourceMapFactory.getInstance().serializeResourceMap(rm);
Document rmDoc = new Document(resourceMapId.getValue(), "http://www.openarchives.org/ore/terms", "", "");
rmDoc.doctext = resourceMapXML;
SystemMetadata resourceMapSysMeta = generateSystemMetadata(rmDoc);
mn.create(session, resourceMapId, IOUtils.toInputStream(resourceMapXML, MetaCatServlet.DEFAULT_ENCODING), resourceMapSysMeta);
}
}
catch(Exception e) {
e.printStackTrace();
System.out.println("Could not create document with id " +
sysmeta.getIdentifier().getValue() + " : " + e.getMessage());
}
finally {
printHeader("Done processing document " + sysmeta.getIdentifier().getValue());
}
}
//logout();
}
/**
* @param doc
* @return
*/
private SystemMetadata generateSystemMetadata(Document doc)
throws Exception {
SystemMetadata sm = new SystemMetadata();
sm.setSerialVersion(BigInteger.valueOf(1));
//set the id
Identifier id = new Identifier();
id.setValue(doc.docid.trim());
sm.setIdentifier(id);
//set the object format
ObjectFormat format = ObjectFormatCache.getInstance().getFormat(doc.doctype);
if (format == null) {
if (doc.doctype.trim().equals("BIN")) {
format = ObjectFormatCache.getInstance().getFormat("application/octet-stream");
}
else {
format = ObjectFormatCache.getInstance().getFormat("text/plain");
}
}
sm.setFormatId(format.getFormatId());
//create the checksum
ByteArrayInputStream bais = new ByteArrayInputStream(doc.doctext.getBytes(MetaCatServlet.DEFAULT_ENCODING));
String algorithm = PropertyService.getProperty("dataone.checksumAlgorithm.default");
Checksum checksum = ChecksumUtil.checksum(bais, algorithm);
sm.setChecksum(checksum);
//set the size
String sizeStr = Long.toString(doc.doctext.getBytes(MetaCatServlet.DEFAULT_ENCODING).length);
sm.setSize(new BigInteger(sizeStr));
//submitter, rights holder
Subject p = new Subject();
p.setValue(subjectDN);
sm.setSubmitter(p);
sm.setRightsHolder(p);
try {
Date dateCreated = parseMetacatDate(doc.createDate);
sm.setDateUploaded(dateCreated);
Date dateUpdated = parseMetacatDate(doc.updateDate);
sm.setDateSysMetadataModified(dateUpdated);
}
catch(Exception e) {
System.out.println("couldn't parse a date: " + e.getMessage());
Date dateCreated = new Date();
sm.setDateUploaded(dateCreated);
Date dateUpdated = new Date();
sm.setDateSysMetadataModified(dateUpdated);
}
NodeReference nr = new NodeReference();
nr.setValue(PropertyService.getProperty("dataone.nodeId"));
sm.setOriginMemberNode(nr);
sm.setAuthoritativeMemberNode(nr);
// create access policy
AccessPolicy accessPolicy = new AccessPolicy();
AccessRule accessRule = new AccessRule();
accessRule.addPermission(Permission.READ);
Subject subject = new Subject();
subject.setValue(Constants.SUBJECT_PUBLIC);
accessRule.addSubject(subject);
accessPolicy.addAllow(accessRule);
sm.setAccessPolicy(accessPolicy);
return sm;
}
private void printHeader(String s) {
System.out.println("****** " + s + " *******");
}
/**
* parse the metacat date which looks like 2010-06-08 (YYYY-MM-DD) into
* a proper date object
* @param date
* @return
*/
private Date parseMetacatDate(String date)
{
String year = date.substring(0, 4);
String month = date.substring(5, 7);
String day = date.substring(8, 10);
Calendar c = Calendar.getInstance();
c.set(new Integer(year).intValue(),
new Integer(month).intValue(),
new Integer(day).intValue());
return c.getTime();
}
/**
* parse a metacat query response and return a vector of docids
* @param response
* @return
*/
private Vector<Document> parseResponse(String response)
{
Vector<Document> v = new Vector<Document>();
int dstart = response.indexOf("<document>");
int dend = response.indexOf("</document>", dstart);
while(dstart != -1)
{
String doc = response.substring(dstart + "<document>".length(), dend);
//System.out.println("adding " + docid);
Document d = new Document(getFieldFromDoc(doc, "docid"),
getFieldFromDoc(doc, "doctype"),
getFieldFromDoc(doc, "createdate"),
getFieldFromDoc(doc, "updatedate"));
v.add(d);
dstart = response.indexOf("<document>", dend);
dend = response.indexOf("</document>", dstart);
}
return v;
}
private String getFieldFromDoc(String doc, String fieldname)
{
String field = "<" + fieldname + ">";
String fieldend = "</" + fieldname + ">";
int start = doc.indexOf(field);
int end = doc.indexOf(fieldend);
String s = doc.substring(start + field.length(), end);
//System.out.println("field: " + fieldname + " : " + s);
return s;
}
/**
* returns a sessionid
* @return
*/
private String login()
throws Exception
{
InputStream is = getResponse(sourceUrl, "/metacat",
"action=login&username=" + username + "&password=" + password + "&qformat=xml", "POST");
String response = IOUtils.toString(is, MetaCatServlet.DEFAULT_ENCODING);
//System.out.println("response: " + response);
if(response.indexOf("sessionId") == -1)
{
throw new Exception("Error logging into " + sourceUrl);
}
String sessionid = response.substring(
response.indexOf("<sessionId>") + "<sessionId>".length(),
response.indexOf("</sessionId>"));
System.out.println("sessionid: " + sessionid);
return sessionid;
}
/**
* logout both the source and destination
* @throws Exception
*/
private void logout()
throws Exception
{
getResponse(sourceUrl, "/metacat", "action=logout&username=" + username, "POST");
}
/**
* get an http response
* @param contextRootUrl
* @param resource
* @param urlParameters
* @param method
* @return
* @throws Exception
*/
private InputStream getResponse(String contextRootUrl, String resource,
String urlParameters, String method)
throws Exception
{
HttpURLConnection connection = null ;
String restURL = contextRootUrl+resource;
if (urlParameters != null) {
if (restURL.indexOf("?") == -1)
restURL += "?";
restURL += urlParameters;
if(restURL.indexOf(" ") != -1)
{
restURL = restURL.replaceAll("\\s", "%20");
}
}
URL u = null;
InputStream content = null;
System.out.println("url: " + restURL);
System.out.println("method: " + method);
u = new URL(restURL);
connection = (HttpURLConnection) u.openConnection();
connection.setDoOutput(true);
connection.setDoInput(true);
connection.setRequestMethod(method);
content = connection.getInputStream();
return content;
}
private class Document
{
public String docid;
public String doctype;
public String createDate;
public String updateDate;
public String doctext;
public Document(String docid, String doctype, String createDate, String updateDate)
{
this.docid = docid.trim();
this.doctype = doctype.trim();
this.createDate = createDate.trim();
this.updateDate = updateDate.trim();
}
}
}