/**
* This work was created by participants in the DataONE project, and is
* jointly copyrighted by participating institutions in DataONE. For
* more information on DataONE, see our web site at http://dataone.org.
*
* Copyright ${year}
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dataone.integration;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.SimpleTimeZone;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.dataone.client.D1Node;
import org.dataone.client.auth.CertificateManager;
import org.dataone.client.v1.CNode;
import org.dataone.client.v1.MNode;
import org.dataone.client.v1.itk.D1Object;
import org.dataone.service.exceptions.IdentifierNotUnique;
import org.dataone.service.exceptions.InsufficientResources;
import org.dataone.service.exceptions.InvalidRequest;
import org.dataone.service.exceptions.InvalidSystemMetadata;
import org.dataone.service.exceptions.InvalidToken;
import org.dataone.service.exceptions.NotAuthorized;
import org.dataone.service.exceptions.NotFound;
import org.dataone.service.exceptions.NotImplemented;
import org.dataone.service.exceptions.ServiceFailure;
import org.dataone.service.exceptions.UnsupportedType;
import org.dataone.service.types.v1.Checksum;
import org.dataone.service.types.v1.Group;
import org.dataone.service.types.v1.Identifier;
import org.dataone.service.types.v1.Node;
import org.dataone.service.types.v1.NodeList;
import org.dataone.service.types.v1.NodeReference;
import org.dataone.service.types.v1.NodeType;
import org.dataone.service.types.v1.ObjectFormatIdentifier;
import org.dataone.service.types.v1.ObjectLocation;
import org.dataone.service.types.v1.ObjectLocationList;
import org.dataone.service.types.v1.Permission;
import org.dataone.service.types.v1.Person;
import org.dataone.service.types.v1.Replica;
import org.dataone.service.types.v1.ReplicationStatus;
import org.dataone.service.types.v1.Service;
import org.dataone.service.types.v1.Session;
import org.dataone.service.types.v1.Subject;
import org.dataone.service.types.v1.SubjectInfo;
import org.dataone.service.types.v1.SubjectList;
import org.dataone.service.types.v1.SystemMetadata;
import org.dataone.service.types.v1.util.AccessUtil;
import org.dataone.service.types.v1.util.ChecksumUtil;
import org.dataone.service.util.Constants;
/**
* Utilities that are useful for generating test data.
*/
public class ExampleUtilities {
public static final String CHECKSUM_ALGORITHM = "MD5";
// common object formats to test
public static final String FORMAT_TEXT_PLAIN = "text/plain";
public static final String FORMAT_TEXT_CSV = "text/csv";
public static final String FORMAT_EML_2_0_0 = "eml://ecoinformatics.org/eml-2.0.0";
public static final String FORMAT_EML_2_0_1 = "eml://ecoinformatics.org/eml-2.0.1";
public static final String FORMAT_EML_2_1_0 = "eml://ecoinformatics.org/eml-2.1.0";
public static final String FORMAT_EML_2_1_1 = "eml://ecoinformatics.org/eml-2.1.1";
public static final String FORMAT_RESOURCE_MAP = "http://www.openarchives.org/ore/terms";
// paths to common science data and metadata examples for the above formats
protected static final String SCIDATA_TEXT_PLAIN = "/d1_testdocs/eml200/IPCC.200802107062739.1";
protected static final String SCIDATA_TEXT_CSV = "/d1_testdocs/eml201/TPT001_018MHP2000R00_20110121.40.1.csv";
protected static final String SCIMETA_EML_2_0_0 = "/d1_testdocs/eml200/dpennington.195.2";
protected static final String SCIMETA_EML_2_0_1 = "/d1_testdocs/eml201/TPT001_018MHP2000R00_20110121.50.1.xml";
protected static final String SCIMETA_EML_2_1_0 = "/d1_testdocs/eml210/peggym.130.4";
// TODO: protected static final String SCIMETA_EML_2_1_1 = "need to get a 2.1.1 test doc";
protected static final String RESOURCE_MAP = "/D1shared/resourceMaps/libclient_java_example_2013_04_15.xml";
protected static final String RESOURCE_MAP_NON_PARSING = "/D1shared/resourceMaps/badCreator_jena_doesnot_parse.rdf";
protected final static String preferredMNId = "c3p0";
/**
* creates the identifier, data inputstream, and sysmetadata for testing purposes
* the rightsHolder is set to the subject of the current certificate (user)
*
* uses a default text/plain data source
*
* @deprecated This method uses a submitter based on the CertificateManager's currently
* loaded certificate
*/
public static Object[] generateTestSciDataPackage(String idString, boolean isPrefix)
throws NoSuchAlgorithmException, NotFound, InvalidRequest, IOException {
return generateTestDataPackage(idString, isPrefix, FORMAT_TEXT_PLAIN);
}
/**
* creates the identifier, data inputstream, and sysmetadata for testing purposes
* the rightsHolder is set to the subject of the current certificate (user)
*
* uses a default text/plain data source
*
* @deprecated This method uses a submitter based on the CertificateManager's currently
* loaded certificate
*/
public static Object[] generateTestSciMetaDataPackage(String idString, boolean isPrefix)
throws NoSuchAlgorithmException, NotFound, InvalidRequest, IOException {
return generateTestDataPackage(idString, isPrefix, FORMAT_EML_2_1_0);
}
/**
* creates the identifier, data inputstream, and sysmetadata for testing purposes
* the rightsHolder is set to the subject of the current certificate (user)
*
* uses a default text/plain data source
*/
public static Object[] generateTestSciDataPackage(String idString, boolean isPrefix, String rightsHolderSubjectName)
throws NoSuchAlgorithmException, NotFound, InvalidRequest, IOException {
return generateTestDataPackage(idString, isPrefix, FORMAT_TEXT_PLAIN, rightsHolderSubjectName);
}
/**
* creates the identifier, data inputstream, and sysmetadata for testing purposes
* the rightsHolder is set to the subject of the current certificate (user)
*
* uses a default text/plain data source
*/
public static Object[] generateTestSciMetaDataPackage(String idString, boolean isPrefix, String rightsHolderSubjectName)
throws NoSuchAlgorithmException, NotFound, InvalidRequest, IOException {
return generateTestDataPackage(idString, isPrefix, FORMAT_EML_2_1_0, rightsHolderSubjectName);
}
/**
* Provides a byte array representation of the object for the specified format
* Accepts any of the types defined as constants in ExampleUtilities. eg:
* ExampleUtilities.FORMAT_EML_2_0_0
* @param formatIDString
* @return
* @throws IOException
*/
public static byte[] getExampleObjectOfType(String formatIDString) throws IOException
{
byte[] contentBytes = null;
InputStream fileStream = null;
// choose a test object file based on the object format passed in
if ( formatIDString == FORMAT_TEXT_PLAIN ) {
fileStream = ExampleUtilities.class.getResourceAsStream(SCIDATA_TEXT_PLAIN);
} else if ( formatIDString == FORMAT_EML_2_0_0 ) {
fileStream = ExampleUtilities.class.getResourceAsStream(SCIMETA_EML_2_0_0);
} else if ( formatIDString == FORMAT_EML_2_0_1 ) {
fileStream = ExampleUtilities.class.getResourceAsStream(SCIMETA_EML_2_0_1);
} else if ( formatIDString == FORMAT_EML_2_1_0 ) {
fileStream = ExampleUtilities.class.getResourceAsStream(SCIMETA_EML_2_1_0);
} else if ( formatIDString == FORMAT_RESOURCE_MAP) {
fileStream = ExampleUtilities.class.getResourceAsStream(RESOURCE_MAP);
//TODO: get an EML 2.1.1 test doc in place
//} else if ( formatIDString == FORMAT_EML_2_1_1 ) {
// fileStream = ExampleUtilities.class.getResourceAsStream(SCIMETA_EML_2_1_1);
}
contentBytes = IOUtils.toByteArray(fileStream);
return contentBytes;
}
/**
* creates the identifier, data inputstream, and sysmetadata for testing purposes
* the rightsHolder is set to the subject of the current certificate (user)
*
* @deprecated This method uses a submitter based on the CertificateManager's currently
* loaded certificate
*/
public static Object[] generateTestDataPackage(String idString,
boolean isPrefix, String formatString)
throws NoSuchAlgorithmException, NotFound, InvalidRequest, IOException {
if (isPrefix) {
idString += generateIdentifier();
}
Identifier guid = new Identifier();
guid.setValue(idString);
byte[] contentBytes = getExampleObjectOfType(formatString);
// figure out who we are
String ownerX500 = idString + "_unknownCert";
try {
X509Certificate certificate = CertificateManager.getInstance()
.loadCertificate();
if (certificate != null) {
ownerX500 = CertificateManager.getInstance().getSubjectDN(
certificate);
// sysMeta.getRightsHolder().setValue(ownerX500);
// sysMeta.getSubmitter().setValue(ownerX500);
}
} catch (Exception e) {
// ignore
}
D1Object d1o = new D1Object(guid, contentBytes, formatString,
ownerX500, "authNode");
SystemMetadata sysMeta = d1o.getSystemMetadata();
// match the submitter as the cert DN
sysMeta.setAccessPolicy(AccessUtil.createSingleRuleAccessPolicy(
new String[] { Constants.SUBJECT_PUBLIC },
new Permission[] { Permission.READ }));
ByteArrayInputStream bis = new ByteArrayInputStream(contentBytes);
return new Object[] { guid, bis, sysMeta };
}
public static Object[] generateTestDataPackage(String idString, boolean isPrefix,
String formatString, String rightsHolderSubjectName) throws NoSuchAlgorithmException,
NotFound, InvalidRequest, IOException {
if (isPrefix) {
idString += generateIdentifier();
}
Identifier guid = new Identifier();
guid.setValue(idString);
byte[] contentBytes = getExampleObjectOfType(formatString);
// CertificateManager.getIn
D1Object d1o = new D1Object(guid, contentBytes, formatString, rightsHolderSubjectName,
"authNode");
SystemMetadata sysMeta = d1o.getSystemMetadata();
sysMeta.setAccessPolicy(AccessUtil.createSingleRuleAccessPolicy(
new String[] { Constants.SUBJECT_PUBLIC }, new Permission[] { Permission.READ }));
Subject rightsHolder = new Subject();
rightsHolder.setValue(rightsHolderSubjectName);
sysMeta.setRightsHolder(rightsHolder);
sysMeta.setSubmitter(rightsHolder);
ByteArrayInputStream bis = new ByteArrayInputStream(contentBytes);
return new Object[] { guid, bis, sysMeta };
}
/**
* Create a unique identifier for testing insert and update.
*
* @return a String identifier based on the current date and time
*/
public static String generateIdentifier() {
return ExampleUtilities.generateTimeString();
}
/** Generate a timestamp for use in IDs. */
private static String generateTimeString()
{
StringBuffer pid = new StringBuffer();
// Create a calendar to get the date formatted properly
String[] ids = TimeZone.getAvailableIDs(-8 * 60 * 60 * 1000);
SimpleTimeZone pdt = new SimpleTimeZone(-8 * 60 * 60 * 1000, ids[0]);
pdt.setStartRule(Calendar.APRIL, 1, Calendar.SUNDAY, 2 * 60 * 60 * 1000);
pdt.setEndRule(Calendar.OCTOBER, -1, Calendar.SUNDAY, 2 * 60 * 60 * 1000);
Calendar calendar = new GregorianCalendar(pdt);
Date trialTime = new Date();
calendar.setTime(trialTime);
pid.append(calendar.get(Calendar.YEAR));
pid.append(calendar.get(Calendar.DAY_OF_YEAR));
pid.append(calendar.get(Calendar.HOUR_OF_DAY));
pid.append(calendar.get(Calendar.MINUTE));
pid.append(calendar.get(Calendar.SECOND));
pid.append(calendar.get(Calendar.MILLISECOND));
return pid.toString();
}
/**
* Generate a list of potential replica target nodes using the capabilities
* of the authoritative node and those of the MNs in the NodeList from the CN
* @param cn the CN providing the NodeList
* @param authNode the authoritative Node for the object
* @return the potential replica list of MNs
*/
protected static List<NodeReference> generatePotentialReplicaNodeList(CNode cn, Node authNode) {
// get the full node list from the cn
NodeList nodeList = null;
List<Node> nodes = null;
// get the node list from the CN
try {
nodeList = cn.listNodes();
nodes = nodeList.getNodeList();
} catch (NotImplemented e) {
e.printStackTrace();
} catch (ServiceFailure e) {
e.printStackTrace();
}
//create the list of potential target nodes
List<NodeReference> potentialNodeList = new ArrayList<NodeReference>();
// verify the versions of replication the authNode supports
List<String> implementedVersions = new ArrayList<String>();
List<Service> origServices = authNode.getServices().getServiceList();
for (Service service : origServices) {
if(service.getName().equals("MNReplication") &&
service.getAvailable()) {
implementedVersions.add(service.getVersion());
}
}
// build the potential list of target nodes
for(Node node : nodes) {
// only add MNs as targets, excluding the authoritative MN and MNs that are not tagged to replicate
if ( (node.getType() == NodeType.MN) && node.isReplicate() &&
!node.getIdentifier().getValue().equals(authNode.getIdentifier().getValue())) {
for (Service service : node.getServices().getServiceList()) {
if(service.getName().equals("MNReplication") &&
implementedVersions.contains(service.getVersion()) &&
service.getAvailable()) {
potentialNodeList.add(node.getIdentifier());
}
}
}
}
return potentialNodeList;
}
/** Generate a SystemMetadata object with bogus data. */
@Deprecated
public static SystemMetadata generateSystemMetadata(
Identifier pid, String objectFormatIdString, InputStream source, String mnIdentifier) {
ObjectFormatIdentifier ofid = new ObjectFormatIdentifier();
ofid.setValue(objectFormatIdString);
if (mnIdentifier == null)
return generateSystemMetadata(pid, ofid, source, preferredMNId);
else
return generateSystemMetadata(pid, ofid, source, mnIdentifier);
}
/** Generate a SystemMetadata object with bogus data. */
@Deprecated
protected static SystemMetadata generateSystemMetadata(
Identifier pid, ObjectFormatIdentifier fmtid, InputStream source) {
return generateSystemMetadata(pid, fmtid, source, preferredMNId);
}
/** Generate a SystemMetadata object with bogus data. */
@Deprecated
protected static SystemMetadata generateSystemMetadata(
Identifier pid, ObjectFormatIdentifier fmtid, InputStream source, String mnIdentifier)
{
SystemMetadata sysmeta = new SystemMetadata();
sysmeta.setIdentifier(pid);
sysmeta.setFormatId(fmtid);
sysmeta.setSize(BigInteger.valueOf(12));
Subject submitter = new Subject();
String dn = "uid=jones,o=NCEAS,dc=ecoinformatics,dc=org";
submitter.setValue(dn);
sysmeta.setSubmitter(submitter);
Subject rightsHolder = new Subject();
rightsHolder.setValue(dn);
sysmeta.setRightsHolder(rightsHolder);
sysmeta.setDateSysMetadataModified(new Date());
sysmeta.setDateUploaded(new Date());
NodeReference originMemberNode = new NodeReference();
originMemberNode.setValue(mnIdentifier);
sysmeta.setOriginMemberNode(originMemberNode);
NodeReference authoritativeMemberNode = new NodeReference();
authoritativeMemberNode.setValue(mnIdentifier);
sysmeta.setAuthoritativeMemberNode(authoritativeMemberNode);
Replica firstReplica = new Replica();
NodeReference replicaNodeReference = new NodeReference();
replicaNodeReference.setValue(mnIdentifier);
firstReplica.setReplicaMemberNode(replicaNodeReference);
firstReplica.setReplicationStatus(ReplicationStatus.COMPLETED);
firstReplica.setReplicaVerified(new Date());
sysmeta.addReplica(firstReplica);
sysmeta.setSerialVersion(BigInteger.valueOf(1));
Checksum checksum = null;
try
{
checksum = ChecksumUtil.checksum(source, CHECKSUM_ALGORITHM);
}
catch (Exception e)
{
// TODO Auto-generated catch block
System.out.println("could not create the checksum for the document: " + e.getMessage());
e.printStackTrace();
}
sysmeta.setChecksum(checksum);
return sysmeta;
}
protected static String extractObjectListTotalAttribute(String ol) {
Pattern pat = Pattern.compile("total=\"\\d+\"");
Matcher mat = pat.matcher(ol);
String totalPattern = null;
if (mat.find())
totalPattern = mat.group();
return totalPattern;
}
protected static SubjectList buildSubjectList(Object persons) {
SubjectList sl = new SubjectList();
for(String pString: (String[])persons) {
Person p = new Person();
Subject s = new Subject();
s.setValue(pString);
p.setSubject(s);
sl.addSubject(p.getSubject());
}
return sl;
}
protected static Subject buildSubject(String subjectValue) {
Subject s = new Subject();
s.setValue(subjectValue);
return s;
}
public static Identifier doCreateNewObject(D1Node d1Node, String idPrefix) throws ServiceFailure,
NotImplemented, InvalidToken, NotAuthorized, IdentifierNotUnique, UnsupportedType,
InsufficientResources, InvalidSystemMetadata, NotFound, InvalidRequest
{
// String principal = "uid%3Dkepler,o%3Dunaffiliated,dc%3Decoinformatics,dc%3Dorg";
Session token = null; // mn.login(principal, "kepler");
String idString = idPrefix + ExampleUtilities.generateIdentifier();
Identifier pid = new Identifier();
pid.setValue(idString);
InputStream objectStream =
new Throwable().getStackTrace()[2].getClass().getResourceAsStream("/d1_testdocs/knb-lter-cdr.329066.1.data");
// InputStream objectStream = Caller.getClass().getResourceAsStream(
// "/d1_testdocs/knb-lter-cdr.329066.1.data");
SystemMetadata sysmeta = ExampleUtilities.generateSystemMetadata(pid, "text/csv", objectStream,null);
Identifier rpid = null;
objectStream =
new Throwable().getStackTrace()[2].getClass().getResourceAsStream("/d1_testdocs/knb-lter-cdr.329066.1.data");
if (d1Node instanceof MNode) {
rpid = ((MNode) d1Node).create(token, pid, objectStream, sysmeta);
} else {
rpid = ((CNode) d1Node).create(token, pid, objectStream, sysmeta);
}
assertThat("checking that returned pid matches given ", pid.getValue(), is(rpid.getValue()));
// mn.setAccessPolicy(token, rpid, ContextAwareTestCaseDataone.buildPublicReadAccessPolicy());
System.out.println("new document created on " + d1Node.getNodeBaseServiceUrl() +
" with pid " + rpid.getValue());
if (d1Node instanceof MNode) {
((MNode) d1Node).get(null,pid);
} else {
((CNode) d1Node).get(null,pid);
}
return rpid;
}
/**
* Utility method for getting a mock session object
*
* @return session - the session object with a Subject set and
*/
public static Session getTestSession() {
Session session = new Session();
String subjectStr = "uid=kepler,o=unaffiliated,dc=ecoinformatics,dc=org";
List<Group> groupList= new ArrayList<Group>();
Group group1 = new Group();
group1.setGroupName("cn=test-group,dc=ecoinformatics,dc=org");
groupList.add(group1);
Group group2 = new Group();
group1.setGroupName("cn=test-group2,dc=ecoinformatics,dc=org");
groupList.add(group2);
Subject subject = new Subject();
subject.setValue(subjectStr);
SubjectInfo subjectInfo = new SubjectInfo();
subjectInfo.setGroupList(groupList);
session.setSubject(subject);
session.setSubjectInfo(subjectInfo);
return session;
}
public Person buildPerson(Subject subject, String familyName,
String givenName, String emailString)
{
String[] badParam = new String[]{};
Person person = new Person();
// try {
// InternetAddress ia = new InternetAddress(emailString, true);
if (emailString == null || emailString.trim().equals(""))
badParam[badParam.length] = "emailString";
if (familyName == null || familyName.trim().equals(""))
badParam[badParam.length] = "familyName";
if (givenName == null || givenName.trim().equals(""))
badParam[badParam.length] = "givenName";
if (subject == null || subject.getValue().equals(""))
badParam[badParam.length] = "subject";
if (badParam.length > 0)
throw new IllegalArgumentException("null or empty string values for parameters: " + badParam);
// } catch (AddressException e) {
// // thrown by IndernetAddress constructor
// }
person.addEmail(emailString);
person.addGivenName(givenName);
person.setFamilyName(familyName);
person.setSubject(subject);
return person;
}
public static int countLocationsWithResolve(CNode cn, Identifier pid) throws InvalidToken, ServiceFailure,
NotAuthorized, NotFound, InvalidRequest, NotImplemented {
ObjectLocationList oll = cn.resolve(null, pid);
List<ObjectLocation> locs = oll.getObjectLocationList();
return locs.toArray().length;
}
}