/**
 * The contents of this file are subject to the license and copyright
 * detailed in the LICENSE and NOTICE files at the root of the source
 * tree and available online at
 *
 * http://www.dspace.org/license/
 */
package org.dspace.ctask.general;

import java.io.InputStream;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.namespace.NamespaceContext;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import javax.xml.XMLConstants;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.log4j.Logger;

import org.dspace.content.MetadataValue;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import org.xml.sax.SAXException;

import org.dspace.authorize.AuthorizeException;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
import org.dspace.core.Constants;
import org.dspace.curate.AbstractCurationTask;
import org.dspace.curate.Curator;
import org.dspace.curate.Mutative;
import org.dspace.curate.Suspendable;

/**
 * MetadataWebService task calls a web service using metadata from
 * passed item to obtain data. Depending on configuration, this
 * data may be assigned to item metadata fields, or just recorded in the
 * task result string. Task succeeds if web service call succeeds and 
 * configured updates occur, fails if task user not authorized or item
 * lacks metadata to call service, and returns error in all other cases
 * (except skip status for non-item objects).
 * Intended use: cataloging tool in workflow and general curation.
 * The task uses a URL 'template' to compose the service call, e.g.
 * 
 * {@code http://www.sherpa.ac.uk/romeo/api29.php?issn=\{dc.identifier.issn\}}
 * 
 * Task will substitute the value of the passed item's metadata field
 * in the {parameter} position. If multiple values are present in the
 * item field, the first value is used.
 * 
 * The task uses another property (the datamap) to determine what data
 * to extract from the service response and how to use it, e.g.
 * 
 * {@code //publisher/name=>dc.publisher,//romeocolour}
 * 
 * Task will evaluate the left-hand side (or entire token) of each
 * comma-separated token in the property as an XPath 1.0 expression into
 * the response document, and if there is a mapping symbol (e.g. {@code '=>'}) and
 * value, it will assign the response document value(s) to the named
 * metadata field in the passed item. If the response document contains
 * multiple values, they will all be assigned to the item field. The
 * mapping symbol governs the nature of metadata field assignment:
 * 
 * {@code '->'} mapping will add to any existing values in the item field
 * {@code '=>'} mapping will replace any existing values in the item field
 * {@code '~>'} mapping will add *only* if item field has no existing values
 * 
 * Unmapped data (without a mapping symbol) will simply be added to the task
 * result string, prepended by the XPath expression (a little prettified).
 * Each label/value pair in the result string is separated by a space, 
 * unless the optional 'separator' property is defined.
 * 
 * A very rudimentary facility for transformation of data is supported, e.g.
 * 
 * {@code http://www.crossref.org/openurl/?id=\{doi:dc.relation.isversionof\}&format=unixref}
 *
 * The 'doi:' prefix will cause the task to look for a 'transform' with that
 * name, which is applied to the metadata value before parameter substitution
 * occurs. Transforms are defined in a task property such as the following:
 * 
 * {@code transform.doi = match 10. trunc 60}
 * 
 * This means exclude the value string up to the occurrence of '10.', then
 * truncate after 60 characters. The only transform functions currently defined:
 * 
 * {@code 'cut' <number>} = remove number leading characters
 * {@code 'trunc' <number>} = remove trailing characters after number length
 * {@code 'match' <pattern>} = start match at pattern
 * {@code 'text' <characters>} = append literal characters (enclose in ' ' when whitespace needed)
 * 
 * If the transform results in an invalid state (e.g. cutting more characters
 * than are in the value), the condition will be logged and the 
 * un-transformed value used.
 *
 * Transforms may also be used in datamaps, e.g.
 * 
 * {@code //publisher/name=>shorten:dc.publisher,//romeocolour}
 *  
 * which would apply the 'shorten' transform to the service response value(s)
 * prior to metadata field assignment.
 *
 * An optional property 'headers' may be defined to stipulate any HTTP headers
 * required in the service call. The property syntax is double-pipe separated headers:
 * 
 * {@code Accept: text/xml||Cache-Control: no-cache}
 * 
 * @author richardrodgers
 */
@Mutative
@Suspendable
public class MetadataWebService extends AbstractCurationTask implements NamespaceContext
{
    /** log4j category */
    private static final Logger log = Logger.getLogger(MetadataWebService.class);
    // transform token parsing pattern
	protected Pattern ttPattern = Pattern.compile("\'([^\']*)\'|(\\S+)");
    // URL of web service with template parameters
	protected String urlTemplate = null;
    // template parameter
	protected String templateParam = null;
    // Item metadata field to use in service call
	protected String lookupField = null;
    // Optional transformation of lookupField
	protected String lookupTransform = null;
    // response data to map/record
	protected List<MetadataWebServiceDataInfo> dataList = null;
    // response document parsing tools
	protected DocumentBuilder docBuilder = null;
    // language for metadata fields assigned
	protected String lang = null;
    // field separator in result string
	protected String fieldSeparator = null;
    // optional XML namespace map
	protected Map<String, String> nsMap = new HashMap<String, String>();
    // optional HTTP headers
	protected Map<String, String> headers = new HashMap<String, String>();
    
    /**
     * Initializes task
     * @param curator  Curator object performing this task
     * @param taskId the configured local name of the task 
     */
    @Override
    public void init(Curator curator, String taskId) throws IOException {
    	super.init(curator, taskId);
    	lang = configurationService.getProperty("default.language");
        String fldSep = taskProperty("separator");
        fieldSeparator = (fldSep != null) ? fldSep : " ";
    	urlTemplate = taskProperty("template");
    	templateParam = urlTemplate.substring(urlTemplate.indexOf("{") + 1,
    			                              urlTemplate.indexOf("}"));
    	String[] parsed = parseTransform(templateParam);
    	lookupField = parsed[0];
    	lookupTransform = parsed[1];
    	dataList = new ArrayList<>();
    	for (String entry : taskArrayProperty("datamap")) {
    		entry = entry.trim();
    		String src = entry;
    		String mapping = null;
    		String field = null;
    		int mapIdx = getMapIndex(entry);
    		if (mapIdx > 0) {
    			src = entry.substring(0, mapIdx);
    			mapping = entry.substring(mapIdx, mapIdx + 2);
    			field = entry.substring(mapIdx + 2);
    		}
    		int slIdx = src.lastIndexOf("/");
        	String label = (slIdx > 0) ? src.substring(slIdx + 1) : src;
    		dataList.add(new MetadataWebServiceDataInfo(this, src, label, mapping, field));
    	}
        String hdrs = taskProperty("headers");
        if (hdrs != null) {
            for (String header : hdrs.split("\\|\\|")) {
                int split = header.indexOf(":");
                headers.put(header.substring(0, split).trim(), header.substring(split + 1).trim());
            }
        }
    	// initialize response document parser
    	DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
    	factory.setNamespaceAware(true);
    	try {
    		docBuilder = factory.newDocumentBuilder();
    	} catch (ParserConfigurationException pcE) {
    		log.error("caught exception: " + pcE);
    		// no point in continuing
    		throw new IOException(pcE.getMessage(), pcE);
    	}
    }
    
    /**
     * Perform the curation task upon passed DSO
     *
     * @param dso the DSpace object
     * @throws IOException if IO error
     */
    @Override
    public int perform(DSpaceObject dso) throws IOException  {
    	
    	int status = Curator.CURATE_SKIP;
    	StringBuilder resultSb = new StringBuilder();
    	
        if (dso.getType() == Constants.ITEM) {
            Item item = (Item)dso;
            String itemId = item.getHandle();
            if (itemId == null) {
            	// we are still in workflow - no handle assigned - try title
            	List<MetadataValue> titleDc = itemService.getMetadata(item, "dc", "title", null, Item.ANY);
            	String title = (titleDc.size() > 0) ? titleDc.get(0).getValue() : "untitled - dbId: " + item.getID();
            	itemId = "Workflow item: " + title;
            } else {
                itemId = "handle: " + itemId;
            }
            resultSb.append(itemId);
            // Only proceed if item has a value for service template parameter
			List<MetadataValue> dcVals = itemService.getMetadataByMetadataString(item, lookupField);
            if (dcVals.size() > 0 && dcVals.get(0).getValue().length() > 0) {
            	String value = transform(dcVals.get(0).getValue(), lookupTransform);
            	status = callService(value, item, resultSb);
            } else {
            	resultSb.append(" lacks metadata value required for service: ").append(lookupField);
            	status = Curator.CURATE_FAIL;
            }
        } else {
           resultSb.append("Object skipped");
        }
        report(resultSb.toString());
        setResult(resultSb.toString());
        return status;
    }
    
	protected int callService(String value, Item item, StringBuilder resultSb) throws IOException {
    	
    	String callUrl = urlTemplate.replaceAll("\\{" + templateParam + "\\}", value);
    	HttpClient client = new DefaultHttpClient();
    	HttpGet req = new HttpGet(callUrl);
        for (Map.Entry<String, String> entry : headers.entrySet()) {
            req.addHeader(entry.getKey(), entry.getValue());
        }
    	HttpResponse resp = client.execute(req);
    	int status = Curator.CURATE_ERROR;
    	int statusCode = resp.getStatusLine().getStatusCode();
    	if (statusCode == HttpStatus.SC_OK) {
    		HttpEntity entity = resp.getEntity();
    		if (entity != null) {
    			// boiler-plate handling taken from Apache 4.1 javadoc
    			InputStream instream = entity.getContent();
                try {
                	Document doc = docBuilder.parse(instream);
                	status = processResponse(doc, item, resultSb);
            	} catch (SAXException saxE) {
            		log.error("caught exception: " + saxE);
            		resultSb.append(" unable to read response document");
                } catch (RuntimeException ex) {
                	// In case of an unexpected exception you may want to abort
                	// the HTTP request in order to shut down the underlying
                	// connection and release it back to the connection manager.
                	req.abort();
                	log.error("caught exception: " + ex);
                	throw ex;
                } finally {
                	// Closing the input stream will trigger connection release
                	instream.close();
                }
                // When HttpClient instance is no longer needed,
                // shut down the connection manager to ensure
                // immediate deallocation of all system resources
                client.getConnectionManager().shutdown();
    		} else {
    			log.error(" obtained no valid service response");
    			resultSb.append("no service response");
    		}
    	} else {
    		log.error("service returned non-OK status: " + statusCode);
    		resultSb.append("no service response");
    	}
    	return status;
    }
    
	protected int processResponse(Document doc, Item item, StringBuilder resultSb) throws IOException {
       	boolean update = false;
       	int status = Curator.CURATE_ERROR;
       	List<String> values = new ArrayList<String>();
       	checkNamespaces(doc);
       	try {
       		for (MetadataWebServiceDataInfo info : dataList) {
       			NodeList nodes = (NodeList)info.getExpr().evaluate(doc, XPathConstants.NODESET);
       			values.clear();
       			// if data found and we are mapping, check assignment policy
       			if (nodes.getLength() > 0 && info.getMapping() != null) {
       				if ("=>".equals(info.getMapping())) {
       					itemService.clearMetadata(Curator.curationContext(), item, info.getSchema(), info.getElement(), info.getQualifier(), Item.ANY);
       				} else if ("~>".equals(info.getMapping())) {
       					if (itemService.getMetadata(item, info.getSchema(), info.getElement(), info.getQualifier(), Item.ANY).size() > 0) {
       						// there are values, so don't overwrite
       						continue;
       					}
       				} else {
       					for (MetadataValue dcVal : itemService.getMetadata(item, info.getSchema(), info.getElement(), info.getQualifier(), Item.ANY)) {
       						values.add(dcVal.getValue());
       					}
       				}
       			}
       			for (int i = 0; i < nodes.getLength(); i++) {
       				Node node = nodes.item(i);
       				String tvalue = transform(node.getFirstChild().getNodeValue(), info.getTransform());
       				// assign to metadata field if mapped && not present
       				if (info.getMapping() != null && ! values.contains(tvalue)) {
       					itemService.addMetadata(Curator.curationContext(), item, info.getSchema(), info.getElement(), info.getQualifier(), lang, tvalue);
       					update = true;
       				}
       				// add to result string in any case
       				resultSb.append(fieldSeparator).append(info.getLabel()).append(": ").append(tvalue);
       			}
       		}
       		// update Item if it has changed
       		if (update) {
				itemService.update(Curator.curationContext(), item);
       		}
       		status = Curator.CURATE_SUCCESS;
       	} catch (AuthorizeException authE) {
    		log.error("caught exception: " + authE);
    		resultSb.append(" not authorized to update");
    		status = Curator.CURATE_FAIL;
       	} catch (SQLException sqlE) {
    		log.error("caught exception: " + sqlE);
    		resultSb.append(" error updating metadata");
       	} catch (XPathExpressionException xpeE) {
    		log.error("caught exception: " + xpeE);
    		resultSb.append(" error reading response document");
       	}
        return status;
    }
    
	protected String transform(String value, String transDef) {
    	if (transDef == null) {
    		return value;
    	}
    	String[] tokens = tokenize(transDef);
    	String retValue = value;
    	for (int i = 0; i < tokens.length; i+= 2) {
    		if ("cut".equals(tokens[i]) || "trunc".equals(tokens[i])) {
    			int index = Integer.parseInt(tokens[i+1]);
    			if (retValue.length() > index) {
    				if ("cut".equals(tokens[i])) {
    					retValue = retValue.substring(index);
    				} else {
    					retValue = retValue.substring(0, index);
    				}
    			} else if ("cut".equals(tokens[i])) {
    				log.error("requested cut: " + index + " exceeds value length");
    				return value;
    			}
    		} else if ("match".equals(tokens[i])) {
    			int index2 = retValue.indexOf(tokens[i+1]);
    			if (index2 > 0) {
    				retValue = retValue.substring(index2);
    			} else {
    				log.error("requested match: " + tokens[i+1] + " failed");
    				return value;
    			}
    		} else if ("text".equals(tokens[i])) {
    			retValue = retValue + tokens[i+1];
    		} else {
    			log.error(" unknown transform operation: " + tokens[i]);
    			return value;
    		}
    	}
    	return retValue;
    }
    
	protected String[] tokenize(String text)  {
    	List<String> list = new ArrayList<String>();
    	Matcher m = ttPattern.matcher(text);
    	while (m.find()) {
    		if (m.group(1) != null) {
    			list.add(m.group(1));
            } else if (m.group(2) != null) {
                list.add(m.group(2));
            }
        }
        return list.toArray(new String[0]);
    }
    
	protected int getMapIndex(String mapping) {
    	int index = mapping.indexOf("->");
    	if (index == -1) {
    		index = mapping.indexOf("=>");
    	}
    	if (index == -1) {
    		index = mapping.indexOf("~>");
    	}
    	return index;
    }

	protected String[] parseTransform(String field) {
    	String[] parsed = new String[2];
    	parsed[0] = field;
       	int txIdx = field.indexOf(":");
    	if (txIdx > 0) {
    		// transform specified
    		String txName = field.substring(0, txIdx);
    		parsed[1] = taskProperty("transform." + txName);
    		if (parsed[1] == null) {
    			log.error("no transform found for: " + txName);
    		}
    		parsed[0] = field.substring(txIdx + 1);
    	}
    	return parsed;
    }

	protected void checkNamespaces(Document document) throws IOException {
    	// skip if already done
    	if (dataList.get(0).getExpr() != null) {
    	    return;
    	}
    	try {
    	    XPath xpath = XPathFactory.newInstance().newXPath();
    	    String prefix = null;
            NamedNodeMap attrs = document.getDocumentElement().getAttributes();
    		for (int i = 0; i < attrs.getLength(); i++) {
    		    Node n = attrs.item(i);
                String name = n.getNodeName();
                // remember if a namespace
                if (name.startsWith("xmlns")) {
                    if (! "xmlns".equals(name)) {
                        // it is a declared (non-default) namespace - capture prefix
                        nsMap.put(name.substring(name.indexOf(":") + 1), n.getNodeValue());
                    } else {
                        // it is the default name space - mint a unique prefix
                        prefix = "pre";
                        nsMap.put(prefix, n.getNodeValue());
                    }
                }
            }
            if (nsMap.size() > 0) {
    		    xpath.setNamespaceContext(this);
            }
    		// now compile the XPath expressions
    		for (MetadataWebServiceDataInfo info : dataList) {
			    info.setExpr(xpath.compile(mangleExpr(info.getXpsrc(), prefix)));
		    }
    	} catch (XPathExpressionException xpeE) {
    		log.error("caught exception: " + xpeE);
        	// no point in continuing
        	throw new IOException(xpeE.getMessage(), xpeE);  			
    	}
    }
    
    protected String mangleExpr(String expr, String prefix) {
    	if (prefix == null) {
    		return expr;
    	}
    	// OK the drill is to prepend all node names with the prefix
    	// *unless* the node name already has a prefix.
    	StringBuilder sb = new StringBuilder();
    	int i = 0;
    	while (i < expr.length()) {
    		if (expr.charAt(i) == '/') {
    			sb.append("/");
    			i++;
    		} else {
    			int next = expr.indexOf("/", i);
    			String token = (next > 0) ? expr.substring(i, next) : expr.substring(i);
    			if (! token.startsWith("@") && token.indexOf(":") < 0) {
    				sb.append(prefix).append(":");
    			}
    			sb.append(token);
    			i += token.length();
    		}
    	}
    	return sb.toString();
    }
    
    // ---- NamespaceContext methods ---- //
    
    @Override
	public String getNamespaceURI(String prefix) {
        if (prefix == null) {
        	throw new NullPointerException("Null prefix");
        } else if ("xml".equals(prefix)) {
        	return XMLConstants.XML_NS_URI;
        }
        String nsURI = nsMap.get(prefix);
        return (nsURI != null) ? nsURI : XMLConstants.NULL_NS_URI;
    }

    @Override
	public String getPrefix(String uri) {
        throw new UnsupportedOperationException();
    }

    @Override
	public Iterator getPrefixes(String uri) {
        throw new UnsupportedOperationException();
    }
}