added missing utils

JAVA/src/main/java/it/bancaditalia/oss/sdmx/util/QueryRunner.java

@@ -0,0 +1,358 @@
+package it.bancaditalia.oss.sdmx.util;
+
+import static it.bancaditalia.oss.sdmx.event.RestSdmxEventListener.NO_OP_LISTENER;
+import static it.bancaditalia.oss.sdmx.util.Configuration.getLanguages;
+import static java.lang.System.lineSeparator;
+import static java.util.stream.Collectors.joining;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.Proxy;
+import java.net.ProxySelector;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.InflaterInputStream;
+import java.util.zip.ZipInputStream;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.xml.stream.XMLEventReader;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+
+import it.bancaditalia.oss.sdmx.client.Parser;
+import it.bancaditalia.oss.sdmx.event.OpenEvent;
+import it.bancaditalia.oss.sdmx.event.RedirectionEvent;
+import it.bancaditalia.oss.sdmx.event.RestSdmxEvent;
+import it.bancaditalia.oss.sdmx.event.RestSdmxEventListener;
+import it.bancaditalia.oss.sdmx.exceptions.SdmxException;
+import it.bancaditalia.oss.sdmx.exceptions.SdmxExceptionFactory;
+import it.bancaditalia.oss.sdmx.exceptions.SdmxIOException;
+import it.bancaditalia.oss.sdmx.exceptions.SdmxRedirectionException;
+
+public class QueryRunner
+{
+	private static final String SOURCE_CLASS = QueryRunner.class.getSimpleName();
+	private static final Logger LOGGER = Configuration.getSdmxLogger();
+
+	private static ProxySelector proxySelector = null;
+	private static HostnameVerifier hostnameVerifier = null;
+	private static RestSdmxEventListener	dataFooterMessageEventListener	= NO_OP_LISTENER;
+	private static RestSdmxEventListener	redirectionEventListener		= NO_OP_LISTENER;
+	private static RestSdmxEventListener	openEventListener				= NO_OP_LISTENER;
+
+	public static <T> T runQuery(Parser<T> parser, URL query, Map<String, String> headers) throws SdmxException
+	{
+		return runQuery(parser, query, null, null, headers);
+	}
+	/**
+	 * Returns a reader over the result of an http query.
+	 *
+	 * @param parser a non-null parser used to get the result from the xml file
+	 * @param query a non-null url to query
+	 * @param headers a non-null map containing headers to use in the REST call
+	 * @return The result of the parsing
+	 * 
+	 * @throws SdmxException
+	 */
+	public static <T> T runQuery(Parser<T> parser, URL query, String provider, String dumpName, Map<String, String> headers) throws SdmxException
+	{
+		final String sourceMethod = "runQuery";
+		LOGGER.entering(SOURCE_CLASS, sourceMethod);
+
+		URLConnection conn = null;
+		URL url = null;
+		LOGGER.log(Level.INFO, "Contacting web service with query: {0}", query);
+
+		try
+		{
+			int code;
+			url = query;
+			URL originalURL = url;
+
+			Proxy proxy = (proxySelector != null ? proxySelector : ProxySelector.getDefault()).select(url.toURI()).get(0);
+			LOGGER.fine("Using proxy: " + proxy);
+
+			openEventListener.onSdmxEvent(new OpenEvent(url, headers != null ? headers.get("Accept") : "*/*", getLanguages(), proxy));
+
+			int redirects = 0;
+			do
+			{
+				conn = url.openConnection(proxy);
+
+				if (conn instanceof HttpsURLConnection && hostnameVerifier != null)
+				{
+					LOGGER.fine("Using custom HostnameVerifier");
+					((HttpsURLConnection) conn).setHostnameVerifier(hostnameVerifier);
+				}
+
+				conn.setConnectTimeout(Configuration.getConnectTimeout());
+				conn.setReadTimeout(Configuration.getReadTimeout());
+
+				if (conn instanceof HttpURLConnection)
+				{
+					((HttpURLConnection) conn).setRequestMethod("GET");
+					((HttpURLConnection) conn).setInstanceFollowRedirects(false);
+					if (headers != null)
+						for (Entry<String, String> e: headers.entrySet())
+							conn.addRequestProperty(e.getKey(), e.getValue());
+				}
+
+				code = conn instanceof HttpURLConnection ? ((HttpURLConnection) conn).getResponseCode() : HttpURLConnection.HTTP_OK;
+
+				// note on PR #227
+				// in some cases in https the proxy will not let the connect pass unless a previous call has been done in http.
+				// this is probably caused by the basic auth being disabled in the java env and it can be fixed 
+				// re-allowing it (-Djdk.http.auth.tunneling.disabledSchemes=)
+
+				if (isRedirection(code))
+				{
+					URL redirection = getRedirectionURL(conn, code);
+//					if (conn instanceof HttpURLConnection)
+//						((HttpURLConnection) conn).disconnect();
+					if (isDowngradingProtocolOnRedirect(originalURL, redirection))
+					{
+						throw new SdmxRedirectionException("Downgrading protocol on redirect from '" + originalURL + "' to '" + redirection + "'");
+					}
+					LOGGER.log(Level.INFO, "Redirecting to: {0}", redirection);
+					RestSdmxEvent event = new RedirectionEvent(url, redirection);
+					redirectionEventListener.onSdmxEvent(event);
+					url = redirection;
+					redirects++;
+				}
+			} while (isRedirection(code) && !(isMaxRedirectionReached(redirects)));
+
+			if (isMaxRedirectionReached(redirects))
+			{
+				throw new SdmxRedirectionException("Max redirection reached");
+			}
+
+			if (code == HttpURLConnection.HTTP_OK)
+			{
+				LOGGER.fine("Connection opened. Code: " + code);
+				InputStream stream = conn.getInputStream();
+				String encoding = conn.getContentEncoding() == null ? "" : conn.getContentEncoding();
+				if (encoding.equalsIgnoreCase("gzip"))
+					stream = new GZIPInputStream(stream);
+				else if (encoding.equalsIgnoreCase("deflate"))
+					stream = new InflaterInputStream(stream);
+				else if (conn.getContentType() != null && conn.getContentType().contains("application/octet-stream"))
+				{
+					stream = new ZipInputStream(stream);
+					((ZipInputStream) stream).getNextEntry();
+				}
+
+				if (Configuration.isDumpXml() && dumpName != null) // skip providers < sdmx v2.1
+				{
+//					String resource = URLDecoder.decode(url.getPath(), StandardCharsets.UTF_8.name()).replaceAll(endpoint.getPath() + "/?", "")
+//							.replaceFirst("/$", "").replaceAll("\\p{Punct}", "_") + ".xml";
+					System.err.println(Configuration.getDumpPrefix());
+					File dumpfilename = Paths.get(Configuration.getDumpPrefix(), provider, dumpName + ".xml").toFile();
+					if (dumpfilename.getParentFile().exists() || dumpfilename.getParentFile().mkdirs())
+					{
+						LOGGER.info("Dumping xml to file " + dumpfilename.getAbsolutePath());
+						stream = new TeeInputStream(stream, dumpfilename);
+					}
+					else
+						LOGGER.warning("Error creating path to dump file: " + dumpfilename);
+				}
+
+				try (Reader reader = new InputStreamReader(stream, StandardCharsets.UTF_8))
+				{
+					XMLInputFactory inputFactory = XMLInputFactory.newFactory();
+					preventXXE(inputFactory);
+					BufferedReader br = skipBOM(reader);
+					// InputStream in = new ByteArrayInputStream(xmlBuffer);
+					XMLEventReader eventReader = inputFactory.createXMLEventReader(br);
+
+					return parser.parse(eventReader, getLanguages());
+				}
+			}
+			else
+			{
+				InputStream stream = ((HttpURLConnection) conn).getErrorStream();
+				String encoding = conn.getContentEncoding() == null ? "" : conn.getContentEncoding();
+				if (encoding.equalsIgnoreCase("gzip"))
+					stream = new GZIPInputStream(stream);
+				else if (encoding.equalsIgnoreCase("deflate"))
+					stream = new InflaterInputStream(stream);
+				String msg = new BufferedReader(new InputStreamReader(stream)).lines().collect(joining(lineSeparator()));
+				LOGGER.severe(msg);
+				SdmxException ex = SdmxExceptionFactory.createRestException(code, null, null);
+				if (conn instanceof HttpURLConnection)
+					((HttpURLConnection) conn).disconnect();
+				throw ex;
+			}
+		}
+		catch (IOException e)
+		{
+			LOGGER.severe("Exception. Class: " + e.getClass().getName() + " - Message: " + e.getMessage());
+			LOGGER.log(Level.FINER, "Exception: ", e);
+			throw SdmxExceptionFactory.wrap(e);
+		}
+		catch (XMLStreamException | URISyntaxException e)
+		{
+			LOGGER.severe("Exception caught parsing query results: " + e.getClass().getSimpleName() + ": " + e.getMessage());
+			LOGGER.log(Level.FINER, "Exception: ", e);
+			throw SdmxExceptionFactory.wrap(e);
+		}
+		finally
+		{
+			if (conn != null && conn instanceof HttpURLConnection)
+				((HttpURLConnection) conn).disconnect();
+		}
+	}
+
+	private static boolean isRedirection(int code)
+	{
+		return (code >= HttpURLConnection.HTTP_MULT_CHOICE && code <= HttpURLConnection.HTTP_SEE_OTHER) || code == 307; // TEMPORARY REDIRECT
+	}
+
+	private static URL getRedirectionURL(URLConnection conn, int code) throws SdmxIOException
+	{
+		String location = conn.getHeaderField("Location");
+		if (location == null || location.isEmpty())
+		{
+			throw new SdmxIOException("The endpoint returned redirect code: " + code + ", but the location was empty.", null);
+		}
+		try
+		{
+			return new URL(location);
+		}
+		catch (MalformedURLException ex)
+		{
+			throw new SdmxIOException("The endpoint returned redirect code: " + code + ", but the location was malformed: '" + location + "'.", null);
+		}
+	}
+
+	// https://www.owasp.org/index.php/XML_External_Entity_(XXE)_Prevention_Cheat_Sheet#XMLInputFactory_.28a_StAX_parser.29
+	private static void preventXXE(XMLInputFactory factory)
+	{
+		if (factory.isPropertySupported(XMLInputFactory.SUPPORT_DTD))
+		{
+			factory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+		}
+		if (factory.isPropertySupported(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES))
+		{
+			factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
+		}
+	}
+
+	private static boolean isMaxRedirectionReached(int redirects)
+	{
+		return redirects > Configuration.getMaxRedirects();
+	}
+
+	/**
+	 * https://en.wikipedia.org/wiki/Downgrade_attack
+	 *
+	 * @param oldUrl
+	 * @param newUrl
+	 * @return
+	 */
+	private static boolean isDowngradingProtocolOnRedirect(URL oldUrl, URL newUrl)
+	{
+		return "https".equalsIgnoreCase(oldUrl.getProtocol()) && !"https".equalsIgnoreCase(newUrl.getProtocol());
+	}
+
+	// some 2.0 providers are apparently adding a BOM
+	private static BufferedReader skipBOM(Reader xmlBuffer) throws SdmxException
+	{
+		BufferedReader br = new BufferedReader(xmlBuffer) {
+			@Override
+			public void close() throws IOException
+			{
+				LOGGER.fine("GenericDataParser::skipBOM: closing stream.");
+				super.close();
+			}
+		};
+		try
+		{
+			// java uses Unicode big endian
+			char[] cbuf = new char[1];
+			// TODO: Source of problems here
+			br.mark(1);
+			br.read(cbuf, 0, 1);
+			LOGGER.fine(String.format("0x%2s", Integer.toHexString(cbuf[0])));
+			if ((byte) cbuf[0] == (byte) 0xfeff)
+			{
+				LOGGER.fine("BOM found and skipped");
+			}
+			else
+			{
+				// TODO: Source of problems here
+				LOGGER.fine("GenericDataParser::skipBOM: Resetting stream.");
+				br.reset();
+			}
+		}
+		catch (IOException e)
+		{
+			throw SdmxExceptionFactory.wrap(e);
+		}
+		return br;
+	}
+
+	public static ProxySelector getProxySelector()
+	{
+		return proxySelector;
+	}
+
+	public static void setProxySelector(ProxySelector proxySelector)
+	{
+		QueryRunner.proxySelector = proxySelector;
+	}
+
+	public static HostnameVerifier getHostnameVerifier()
+	{
+		return hostnameVerifier;
+	}
+
+	public static void setHostnameVerifier(HostnameVerifier hostnameVerifier)
+	{
+		QueryRunner.hostnameVerifier = hostnameVerifier;
+	}
+
+	public static RestSdmxEventListener getDataFooterMessageEventListener()
+	{
+		return dataFooterMessageEventListener;
+	}
+
+	public static void setDataFooterMessageEventListener(RestSdmxEventListener dataFooterMessageEventListener)
+	{
+		QueryRunner.dataFooterMessageEventListener = dataFooterMessageEventListener;
+	}
+
+	public static RestSdmxEventListener getRedirectionEventListener()
+	{
+		return redirectionEventListener;
+	}
+
+	public static void setRedirectionEventListener(RestSdmxEventListener redirectionEventListener)
+	{
+		QueryRunner.redirectionEventListener = redirectionEventListener;
+	}
+
+	public static RestSdmxEventListener getOpenEventListener()
+	{
+		return openEventListener;
+	}
+
+	public static void setOpenEventListener(RestSdmxEventListener openEventListener)
+	{
+		QueryRunner.openEventListener = openEventListener;
+	}
+}

JAVA/src/main/java/it/bancaditalia/oss/sdmx/util/TeeInputStream.java

@@ -0,0 +1,37 @@
+package it.bancaditalia.oss.sdmx.util;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public class TeeInputStream extends InputStream
+{
+	private final InputStream source;
+	private final OutputStream teed;
+
+	public TeeInputStream(InputStream source, File tee) throws FileNotFoundException
+	{
+		this.source = source;
+		this.teed = new FileOutputStream(tee);
+	}
+
+	@Override
+	public void close() throws IOException
+	{
+		source.close();
+	}
+
+	@Override
+	public int read() throws IOException
+	{
+		int c = source.read();
+		if (c >= 0)
+			teed.write(c);
+		if (c == '\n')
+			teed.flush();
+		return c;
+	}
+}