/*
 * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringReader;
import java.io.StringWriter;
import java.lang.annotation.Annotation;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.ref.SoftReference;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.UnsupportedCharsetException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;


/**
 * Test framework for running javadoc and performing tests on the resulting output.
 *
 * <p>
 * Tests are typically written as subtypes of JavadocTester, with a main
 * method that creates an instance of the test class and calls the runTests()
 * method. The runTests() methods calls all the test methods declared in the class,
 * and then calls a method to print a summary, and throw an exception if
 * any of the test methods reported a failure.
 *
 * <p>
 * Test methods are identified with a @Test annotation. They have no parameters.
 * The name of the method is not important, but if you have more than one, it is
 * recommended that the names be meaningful and suggestive of the test case
 * contained therein.
 *
 * <p>
 * Typically, a test method will invoke javadoc, and then perform various
 * checks on the results. The standard checks are:
 *
 * <dl>
 * <dt>checkExitCode
 * <dd>Check the exit code returned from javadoc.
 * <dt>checkOutput
 * <dd>Perform a series of checks on the contents on a file or output stream
 *     generated by javadoc.
 *     The checks can be either that a series of strings are found or are not found.
 * <dt>checkFiles
 * <dd>Perform a series of checks on the files generated by javadoc.
 *     The checks can be that a series of files are found or are not found.
 * </dl>
 *
 * <pre><code>
 *  public class MyTester extends JavadocTester {
 *      public static void main(String... args) throws Exception {
 *          MyTester tester = new MyTester();
 *          tester.runTests();
 *      }
 *
 *      // test methods...
 *      {@literal @}Test
 *      void test() {
 *          javadoc(<i>args</i>);
 *          checkExit(Exit.OK);
 *          checkOutput(<i>file</i>, true,
 *              <i>strings-to-find</i>);
 *          checkOutput(<i>file</i>, false,
 *              <i>strings-to-not-find</i>);
 *      }
 *  }
 * </code></pre>
 *
 * <p>
 * If javadoc is run more than once in a test method, you can compare the
 * results that are generated with the diff method. Since files written by
 * javadoc typically contain a timestamp, you may want to use the -notimestamp
 * option if you are going to compare the results from two runs of javadoc.
 *
 * <p>
 * If you have many calls of checkOutput that are very similar, you can write
 * your own check... method to reduce the amount of duplication. For example,
 * if you want to check that many files contain the same string, you could
 * write a method that takes a varargs list of files and calls checkOutput
 * on each file in turn with the string to be checked.
 *
 * <p>
 * You can also write you own custom check methods, which can use
 * readFile to get the contents of a file generated by javadoc,
 * and then use pass(...) or fail(...) to report whether the check
 * succeeded or not.
 *
 * <p>
 * You can have many separate test methods, each identified with a @Test
 * annotation. However, you should <b>not</b> assume they will be called
 * in the order declared in your source file.  If the order of a series
 * of javadoc invocations is important, do that within a single method.
 * If the invocations are independent, for better clarity, use separate
 * test methods, each with their own set of checks on the results.
 *
 * @author Doug Kramer
 * @author Jamie Ho
 * @author Jonathan Gibbons (rewrite)
 */
public abstract class JavadocTester {

    public static final String FS = System.getProperty("file.separator");
    public static final String PS = System.getProperty("path.separator");
    public static final String NL = System.getProperty("line.separator");
    public static final Path currDir = Paths.get(".").toAbsolutePath().normalize();

    public enum Output {
        /** The name of the output stream from javadoc. */
        OUT,
        /** The name for any output written to System.out. */
        STDOUT,
        /** The name for any output written to System.err. */
        STDERR
    }

    /** The output directory used in the most recent call of javadoc. */
    protected File outputDir;

    /** The output charset used in the most recent call of javadoc. */
    protected Charset charset = Charset.defaultCharset();

    /** The exit code of the most recent call of javadoc. */
    private int exitCode;

    /** The output generated by javadoc to the various writers and streams. */
    private final Map<Output, String> outputMap = new EnumMap<>(Output.class);

    /** A cache of file content, to avoid reading files unnecessarily. */
    private final Map<File,SoftReference<String>> fileContentCache = new HashMap<>();
    /** The charset used for files in the fileContentCache. */
    private Charset fileContentCacheCharset = null;

    /** Stream used for logging messages. */
    protected final PrintStream out = System.out;

    /** The directory containing the source code for the test. */
    public static final String testSrc = System.getProperty("test.src");

    /**
     * Get the path for a source file in the test source directory.
     * @param path the path of a file or directory in the source directory
     * @return the full path of the specified file
     */
    public static String testSrc(String path) {
        return new File(testSrc, path).getPath();
    }

    /**
     * Alternatives for checking the contents of a directory.
     */
    public enum DirectoryCheck {
        /**
         * Check that the directory is empty.
         */
        EMPTY((file, name) -> true),
        /**
         * Check that the directory does not contain any HTML files,
         * such as may have been generated by a prior run of javadoc
         * using this directory.
         * For now, the check is only performed on the top level directory.
         */
        NO_HTML_FILES((file, name) -> name.endsWith(".html")),
        /**
         * No check is performed on the directory contents.
         */
        NONE(null) { @Override void check(File dir) { } };

        /** The filter used to detect that files should <i>not</i> be present. */
        FilenameFilter filter;

        DirectoryCheck(FilenameFilter f) {
            filter = f;
        }

        void check(File dir) {
            if (dir.isDirectory()) {
                String[] contents = dir.list(filter);
                if (contents == null)
                    throw new Error("cannot list directory: " + dir);
                if (contents.length > 0) {
                    System.err.println("Found extraneous files in dir:" + dir.getAbsolutePath());
                    for (String x : contents) {
                        System.err.println(x);
                    }
                    throw new Error("directory has unexpected content: " + dir);
                }
            }
        }
    }

    private DirectoryCheck outputDirectoryCheck = DirectoryCheck.EMPTY;

    private boolean automaticCheckLinks = true;

    /** The current subtest number. Incremented when checking(...) is called. */
    private int numTestsRun = 0;

    /** The number of subtests passed. Incremented when passed(...) is called. */
    private int numTestsPassed = 0;

    /** The current run of javadoc. Incremented when javadoc is called. */
    private int javadocRunNum = 0;

    /** The current subtest number for this run of javadoc. Incremented when checking(...) is called. */
    private int javadocTestNum = 0;

    /** Marker annotation for test methods to be invoked by runTests. */
    @Retention(RetentionPolicy.RUNTIME)
    @interface Test { }

    /**
     * Run all methods annotated with @Test, followed by printSummary.
     * Typically called on a tester object in main()
     * @throws Exception if any errors occurred
     */
    public void runTests() throws Exception {
        runTests(m -> new Object[0]);
    }

    /**
     * Run all methods annotated with @Test, followed by printSummary.
     * Typically called on a tester object in main()
     * @param f a function which will be used to provide arguments to each
     *          invoked method
     * @throws Exception if any errors occurred
     */
    public void runTests(Function<Method, Object[]> f) throws Exception {
        for (Method m: getClass().getDeclaredMethods()) {
            Annotation a = m.getAnnotation(Test.class);
            if (a != null) {
                try {
                    out.println("Running test " + m.getName());
                    m.invoke(this, f.apply(m));
                } catch (InvocationTargetException e) {
                    Throwable cause = e.getCause();
                    throw (cause instanceof Exception) ? ((Exception) cause) : e;
                }
                out.println();
            }
        }
        printSummary();
    }

    /**
     * Run javadoc.
     * The output directory used by this call and the final exit code
     * will be saved for later use.
     * To aid the reader, it is recommended that calls to this method
     * put each option and the arguments it takes on a separate line.
     *
     * Example:
     * <pre><code>
     *  javadoc("-d", "out",
     *          "-sourcepath", testSrc,
     *          "-notimestamp",
     *          "pkg1", "pkg2", "pkg3/C.java");
     * </code></pre>
     *
     * @param args the arguments to pass to javadoc
     */
    public void javadoc(String... args) {
        outputMap.clear();
        fileContentCache.clear();

        javadocRunNum++;
        javadocTestNum = 0; // reset counter for this run of javadoc
        if (javadocRunNum == 1) {
            out.println("Running javadoc...");
        } else {
            out.println("Running javadoc (run "+ javadocRunNum + ")...");
        }

        outputDir = new File(".");
        String charsetArg = null;
        String docencodingArg = null;
        String encodingArg = null;
        for (int i = 0; i < args.length - 2; i++) {
            switch (args[i]) {
                case "-d":
                    outputDir = new File(args[++i]);
                    break;
                case "-charset":
                    charsetArg = args[++i];
                    break;
                case "-docencoding":
                    docencodingArg = args[++i];
                    break;
                case "-encoding":
                    encodingArg = args[++i];
                    break;
            }
        }

        // The following replicates HtmlConfiguration.finishOptionSettings0
        // and sets up the charset used to read files.
        String cs;
        if (docencodingArg == null) {
            if (charsetArg == null) {
                cs = (encodingArg == null) ? "UTF-8" : encodingArg;
            } else {
                cs = charsetArg;
            }
        } else {
           cs = docencodingArg;
        }
        try {
            charset = Charset.forName(cs);
        } catch (UnsupportedCharsetException e) {
            charset = Charset.defaultCharset();
        }

        out.println("args: " + Arrays.toString(args));
//        log.setOutDir(outputDir);

        outputDirectoryCheck.check(outputDir);

        // This is the sole stream used by javadoc
        WriterOutput outOut = new WriterOutput();

        // These are to catch output to System.out and System.err,
        // in case these are used instead of the primary streams
        StreamOutput sysOut = new StreamOutput(System.out, System::setOut);
        StreamOutput sysErr = new StreamOutput(System.err, System::setErr);

        try {
            exitCode = jdk.javadoc.internal.tool.Main.execute(args, outOut.pw);
        } finally {
            outputMap.put(Output.STDOUT, sysOut.close());
            outputMap.put(Output.STDERR, sysErr.close());
            outputMap.put(Output.OUT, outOut.close());
        }

        outputMap.forEach((name, text) -> {
            if (!text.isEmpty()) {
                out.println("javadoc " + name + ":");
                out.println(text);
            }
        });

        if (automaticCheckLinks && exitCode == Exit.OK.code && outputDir.exists()) {
            checkLinks();
        }
    }

    /**
     * Set the kind of check for the initial contents of the output directory
     * before javadoc is run.
     * The filter should return true for files that should <b>not</b> appear.
     * @param c the kind of check to perform
     */
    public void setOutputDirectoryCheck(DirectoryCheck c) {
        outputDirectoryCheck = c;
    }

    /**
     * Set whether or not to perform an automatic call of checkLinks.
     */
    public void setAutomaticCheckLinks(boolean b) {
        automaticCheckLinks = b;
    }

    /**
     * The exit codes returned by the javadoc tool.
     * @see jdk.javadoc.internal.tool.Main.Result
     */
    public enum Exit {
        OK(0),        // Javadoc completed with no errors.
        ERROR(1),     // Completed but reported errors.
        CMDERR(2),    // Bad command-line arguments
        SYSERR(3),    // System error or resource exhaustion.
        ABNORMAL(4);  // Javadoc terminated abnormally

        Exit(int code) {
            this.code = code;
        }

        final int code;

        @Override
        public String toString() {
            return name() + '(' + code + ')';
        }
    }

    /**
     * Check the exit code of the most recent call of javadoc.
     *
     * @param expected the exit code that is required for the test
     * to pass.
     */
    public void checkExit(Exit expected) {
        checking("check exit code");
        if (exitCode == expected.code) {
            passed("return code " + exitCode);
        } else {
            failed("return code " + exitCode +"; expected " + expected);
        }
    }

    /**
     * Check for content in (or not in) the generated output.
     * Within the search strings, the newline character \n
     * will be translated to the platform newline character sequence.
     * @param path a path within the most recent output directory
     *  or the name of one of the output buffers, identifying
     *  where to look for the search strings.
     * @param expectedFound true if all of the search strings are expected
     *  to be found, or false if the file is not expected to be found
     * @param strings the strings to be searched for
     */
    public void checkFileAndOutput(String path, boolean expectedFound, String... strings) {
        if (expectedFound) {
            checkOutput(path, true, strings);
        } else {
            checkFiles(false, path);
        }
    }

    /**
     * Check for content in (or not in) the generated output.
     * Within the search strings, the newline character \n
     * will be translated to the platform newline character sequence.
     * @param path a path within the most recent output directory, identifying
     *  where to look for the search strings.
     * @param expectedFound true if all of the search strings are expected
     *  to be found, or false if all of the strings are expected to be
     *  not found
     * @param strings the strings to be searched for
     */
    public void checkOutput(String path, boolean expectedFound, String... strings) {
        // Read contents of file
        try {
            String fileString = readFile(outputDir, path);
            checkOutput(new File(outputDir, path).getPath(), fileString, expectedFound, strings);
        } catch (Error e) {
            checking("Read file");
            failed("Error reading file: " + e);
        }
    }

    /**
     * Check for content in (or not in) the one of the output streams written by
     * javadoc. Within the search strings, the newline character \n
     * will be translated to the platform newline character sequence.
     * @param output the output stream to check
     * @param expectedFound true if all of the search strings are expected
     *  to be found, or false if all of the strings are expected to be
     *  not found
     * @param strings the strings to be searched for
     */
    public void checkOutput(Output output, boolean expectedFound, String... strings) {
        checkOutput(output.toString(), outputMap.get(output), expectedFound, strings);
    }

    // NOTE: path may be the name of an Output stream as well as a file path
    private void checkOutput(String path, String fileString, boolean expectedFound, String... strings) {
        for (String stringToFind : strings) {
//            log.logCheckOutput(path, expectedFound, stringToFind);
            checking("checkOutput");
            // Find string in file's contents
            boolean isFound = findString(fileString, stringToFind);
            if (isFound == expectedFound) {
                passed(path + ": following text " + (isFound ? "found:" : "not found:") + "\n"
                        + stringToFind);
            } else {
                failed(path + ": following text " + (isFound ? "found:" : "not found:") + "\n"
                        + stringToFind + '\n' +
                        "found \n" +
                        fileString);
            }
        }
    }

    public void checkLinks() {
        checking("Check links");
        LinkChecker c = new LinkChecker(out, this::readFile);
        try {
            c.checkDirectory(outputDir.toPath());
            c.report();
            int errors = c.getErrorCount();
            if (errors == 0) {
                passed("Links are OK");
            } else {
                failed(errors + " errors found when checking links");
            }
        } catch (IOException e) {
            failed("exception thrown when reading files: " + e);
        }
    }

    /**
     * Get the content of the one of the output streams written by javadoc.
     * @param output the name of the output stream
     * @return the content of the output stream
     */
    public String getOutput(Output output) {
        return outputMap.get(output);
    }

    /**
     * Get the content of the one of the output streams written by javadoc.
     * @param output the name of the output stream
     * @return the content of the output stream, as a line of lines
     */
    public List<String> getOutputLines(Output output) {
        String text = outputMap.get(output);
        return (text == null) ? Collections.emptyList() : Arrays.asList(text.split(NL));
    }

    /**
     * Check for files in (or not in) the generated output.
     * @param expectedFound true if all of the files are expected
     *  to be found, or false if all of the files are expected to be
     *  not found
     * @param paths the files to check, within the most recent output directory.
     * */
    public void checkFiles(boolean expectedFound, String... paths) {
        checkFiles(expectedFound, Arrays.asList(paths));
    }

    /**
     * Check for files in (or not in) the generated output.
     * @param expectedFound true if all of the files are expected
     *  to be found, or false if all of the files are expected to be
     *  not found
     * @param paths the files to check, within the most recent output directory.
     * */
    public void checkFiles(boolean expectedFound, Collection<String> paths) {
        for (String path: paths) {
//            log.logCheckFile(path, expectedFound);
            checking("checkFile");
            File file = new File(outputDir, path);
            boolean isFound = file.exists();
            if (isFound == expectedFound) {
                passed(file, "file " + (isFound ? "found:" : "not found:") + "\n");
            } else {
                failed(file, "file " + (isFound ? "found:" : "not found:") + "\n");
            }
        }
    }

    /**
     * Check that a series of strings are found in order in a file in
     * the generated output.
     * @param path the file to check
     * @param strings  the strings whose order to check
     */
    public void checkOrder(String path, String... strings) {
        File file = new File(outputDir, path);
        String fileString = readOutputFile(path);
        int prevIndex = -1;
        for (String s : strings) {
            s = s.replace("\n", NL); // normalize new lines
            int currentIndex = fileString.indexOf(s, prevIndex + 1);
            checking("file: " + file + ": " + s + " at index " + currentIndex);
            if (currentIndex == -1) {
                failed(file, s + " not found.");
                continue;
            }
            if (currentIndex > prevIndex) {
                passed(file, s + " is in the correct order");
            } else {
                failed(file, s + " is in the wrong order.");
            }
            prevIndex = currentIndex;
        }
    }

    /**
     * Ensures that a series of strings appear only once, in the generated output,
     * noting that, this test does not exhaustively check for all other possible
     * duplicates once one is found.
     * @param path the file to check
     * @param strings ensure each are unique
     */
    public void checkUnique(String path, String... strings) {
        File file = new File(outputDir, path);
        String fileString = readOutputFile(path);
        for (String s : strings) {
            int currentIndex = fileString.indexOf(s);
            checking(s + " at index " + currentIndex);
            if (currentIndex == -1) {
                failed(file, s + " not found.");
                continue;
            }
            int nextindex = fileString.indexOf(s, currentIndex + s.length());
            if (nextindex == -1) {
                passed(file, s + " is unique");
            } else {
                failed(file, s + " is not unique, found at " + nextindex);
            }
        }
    }

    /**
     * Compare a set of files in each of two directories.
     *
     * @param baseDir1 the directory containing the first set of files
     * @param baseDir2 the directory containing the second set of files
     * @param files the set of files to be compared
     */
    public void diff(String baseDir1, String baseDir2, String... files) {
        File bd1 = new File(baseDir1);
        File bd2 = new File(baseDir2);
        for (String file : files) {
            diff(bd1, bd2, file);
        }
    }

    /**
     * A utility to copy a directory from one place to another.
     *
     * @param targetDir the directory to copy.
     * @param destDir the destination to copy the directory to.
     */
    // TODO: convert to using java.nio.Files.walkFileTree
    public void copyDir(String targetDir, String destDir) {
        try {
            File targetDirObj = new File(targetDir);
            File destDirParentObj = new File(destDir);
            File destDirObj = new File(destDirParentObj, targetDirObj.getName());
            if (! destDirParentObj.exists()) {
                destDirParentObj.mkdir();
            }
            if (! destDirObj.exists()) {
                destDirObj.mkdir();
            }
            String[] files = targetDirObj.list();
            for (String file : files) {
                File srcFile = new File(targetDirObj, file);
                File destFile = new File(destDirObj, file);
                if (srcFile.isFile()) {
                    out.println("Copying " + srcFile + " to " + destFile);
                    copyFile(destFile, srcFile);
                } else if(srcFile.isDirectory()) {
                    copyDir(srcFile.getAbsolutePath(), destDirObj.getAbsolutePath());
                }
            }
        } catch (IOException exc) {
            throw new Error("Could not copy " + targetDir + " to " + destDir);
        }
    }

    /**
     * Copy source file to destination file.
     *
     * @param destfile the destination file
     * @param srcfile the source file
     * @throws IOException
     */
    public void copyFile(File destfile, File srcfile) throws IOException {
        Files.copy(srcfile.toPath(), destfile.toPath());
    }

    /**
     * Read a file from the output directory.
     *
     * @param fileName  the name of the file to read
     * @return          the file in string format
     */
    public String readOutputFile(String fileName) throws Error {
        return readFile(outputDir, fileName);
    }

    protected String readFile(String fileName) throws Error {
        return readFile(outputDir, fileName);
    }

    protected String readFile(String baseDir, String fileName) throws Error {
        return readFile(new File(baseDir), fileName);
    }

    private String readFile(Path file) {
        File baseDir;
        if (file.startsWith(outputDir.toPath())) {
            baseDir = outputDir;
        } else if (file.startsWith(currDir)) {
            baseDir = currDir.toFile();
        } else {
            baseDir = file.getParent().toFile();
        }
        String fileName = baseDir.toPath().relativize(file).toString();
        return readFile(baseDir, fileName);
    }

    /**
     * Read the file and return it as a string.
     *
     * @param baseDir   the directory in which to locate the file
     * @param fileName  the name of the file to read
     * @return          the file in string format
     */
    private String readFile(File baseDir, String fileName) throws Error {
        if (!Objects.equals(fileContentCacheCharset, charset)) {
            fileContentCache.clear();
            fileContentCacheCharset = charset;
        }
        try {
            File file = new File(baseDir, fileName);
            SoftReference<String> ref = fileContentCache.get(file);
            String content = (ref == null) ? null : ref.get();
            if (content != null)
                return content;

            // charset defaults to a value inferred from latest javadoc run
            content = new String(Files.readAllBytes(file.toPath()), charset);
            fileContentCache.put(file, new SoftReference<>(content));
            return content;
        } catch (FileNotFoundException e) {
            throw new Error("File not found: " + fileName + ": " + e);
        } catch (IOException e) {
            throw new Error("Error reading file: " + fileName + ": " + e);
        }
    }

    protected void checking(String message) {
        numTestsRun++;
        javadocTestNum++;
        print("Starting subtest " + javadocRunNum + "." + javadocTestNum, message);
    }

    protected void passed(File file, String message) {
        passed(file + ": " + message);
    }

    protected void passed(String message) {
        numTestsPassed++;
        print("Passed", message);
        out.println();
    }

    protected void failed(File file, String message) {
        failed(file + ": " + message);
    }

    protected void failed(String message) {
        print("FAILED", message);
        StackWalker.getInstance().walk(s -> {
            s.dropWhile(f -> f.getMethodName().equals("failed"))
                    .takeWhile(f -> !f.getMethodName().equals("runTests"))
                    .forEach(f -> out.println("        at "
                            + f.getClassName() + "." + f.getMethodName()
                            + "(" + f.getFileName() + ":" + f.getLineNumber() + ")"));
            return null;
        });
        out.println();
    }

    private void print(String prefix, String message) {
        if (message.isEmpty())
            out.println(prefix);
        else {
            out.print(prefix);
            out.print(": ");
            out.print(message.replace("\n", NL));
            if (!(message.endsWith("\n") || message.endsWith(NL))) {
                out.println();
            }
        }
    }

    /**
     * Print a summary of the test results.
     */
    protected void printSummary() {
        String javadocRuns = (javadocRunNum <= 1) ? ""
                : ", in " + javadocRunNum + " runs of javadoc";

        if (numTestsRun != 0 && numTestsPassed == numTestsRun) {
            // Test passed
            out.println();
            out.println("All " + numTestsPassed + " subtests passed" + javadocRuns);
        } else {
            // Test failed
            throw new Error((numTestsRun - numTestsPassed)
                    + " of " + (numTestsRun)
                    + " subtests failed"
                    + javadocRuns);
        }
    }

    /**
     * Search for the string in the given file and return true
     * if the string was found.
     *
     * @param fileString    the contents of the file to search through
     * @param stringToFind  the string to search for
     * @return              true if the string was found
     */
    private boolean findString(String fileString, String stringToFind) {
        // javadoc (should) always use the platform newline sequence,
        // but in the strings to find it is more convenient to use the Java
        // newline character. So we translate \n to NL before we search.
        stringToFind = stringToFind.replace("\n", NL);
        return fileString.contains(stringToFind);
    }

    /**
     * Compare the two given files.
     *
     * @param baseDir1 the directory in which to locate the first file
     * @param baseDir2 the directory in which to locate the second file
     * @param file the file to compare in the two base directories
     * @param throwErrorIFNoMatch flag to indicate whether or not to throw
     * an error if the files do not match.
     * @return true if the files are the same and false otherwise.
     */
    private void diff(File baseDir1, File baseDir2, String file) {
        String file1Contents = readFile(baseDir1, file);
        String file2Contents = readFile(baseDir2, file);
        checking("diff " + new File(baseDir1, file) + ", " + new File(baseDir2, file));
        if (file1Contents.trim().compareTo(file2Contents.trim()) == 0) {
            passed("files are equal");
        } else {
            failed("files differ");
        }
    }

    /**
     * Utility class to simplify the handling of temporarily setting a
     * new stream for System.out or System.err.
     */
    private static class StreamOutput {
        // functional interface to set a stream.
        private interface Initializer {
            void set(PrintStream s);
        }

        private final ByteArrayOutputStream baos = new ByteArrayOutputStream();
        private final PrintStream ps = new PrintStream(baos);
        private final PrintStream prev;
        private final Initializer init;

        StreamOutput(PrintStream s, Initializer init) {
            prev = s;
            init.set(ps);
            this.init = init;
        }

        String close() {
            init.set(prev);
            ps.close();
            return baos.toString();
        }
    }

    /**
     * Utility class to simplify the handling of creating an in-memory PrintWriter.
     */
    private static class WriterOutput {
        private final StringWriter sw = new StringWriter();
        final PrintWriter pw = new PrintWriter(sw);
        String close() {
            pw.close();
            return sw.toString();
        }
    }


//    private final Logger log = new Logger();

    //--------- Logging --------------------------------------------------------
    //
    // This class writes out the details of calls to checkOutput and checkFile
    // in a canonical way, so that the resulting file can be checked against
    // similar files from other versions of JavadocTester using the same logging
    // facilities.

    static class Logger {
        private static final int PREFIX = 40;
        private static final int SUFFIX = 20;
        private static final int MAX = PREFIX + SUFFIX;
        List<String> tests = new ArrayList<>();
        String outDir;
        String rootDir = rootDir();

        static String rootDir() {
            File f = new File(".").getAbsoluteFile();
            while (!new File(f, ".hg").exists())
                f = f.getParentFile();
            return f.getPath();
        }

        void setOutDir(File outDir) {
            this.outDir = outDir.getPath();
        }

        void logCheckFile(String file, boolean positive) {
            // Strip the outdir because that will typically not be the same
            if (file.startsWith(outDir + "/"))
                file = file.substring(outDir.length() + 1);
            tests.add(file + " " + positive);
        }

        void logCheckOutput(String file, boolean positive, String text) {
            // Compress the string to be displayed in the log file
            String simpleText = text.replaceAll("\\s+", " ").replace(rootDir, "[ROOT]");
            if (simpleText.length() > MAX)
                simpleText = simpleText.substring(0, PREFIX)
                        + "..." + simpleText.substring(simpleText.length() - SUFFIX);
            // Strip the outdir because that will typically not be the same
            if (file.startsWith(outDir + "/"))
                file = file.substring(outDir.length() + 1);
            // The use of text.hashCode ensure that all of "text" is taken into account
            tests.add(file + " " + positive + " " + text.hashCode() + " " + simpleText);
        }

        void write() {
            // sort the log entries because the subtests may not be executed in the same order
            tests.sort((a, b) -> a.compareTo(b));
            try (BufferedWriter bw = new BufferedWriter(new FileWriter("tester.log"))) {
                for (String t: tests) {
                    bw.write(t);
                    bw.newLine();
                }
            } catch (IOException e) {
                throw new Error("problem writing log: " + e);
            }
        }
    }

    // Support classes for checkLinks

    /**
     * A basic HTML parser. Override the protected methods as needed to get notified
     * of significant items in any file that is read.
     */
    static abstract class HtmlParser {

        protected final PrintStream out;
        protected final Function<Path,String> fileReader;

        private Path file;
        private StringReader in;
        private int ch;
        private int lineNumber;
        private boolean inScript;
        private boolean xml;

        HtmlParser(PrintStream out, Function<Path,String> fileReader) {
            this.out = out;
            this.fileReader = fileReader;
        }

        /**
         * Read a file.
         * @param file the file to be read
         * @throws IOException if an error occurs while reading the file
         */
        void read(Path file) throws IOException {
            try (StringReader r = new StringReader(fileReader.apply(file))) {
                this.file = file;
                this.in = r;

                startFile(file);
                try {
                    lineNumber = 1;
                    xml = false;
                    nextChar();

                    while (ch != -1) {
                        switch (ch) {

                            case '<':
                                html();
                                break;

                            default:
                                nextChar();
                        }
                    }
                } finally {
                    endFile();
                }
            } catch (IOException e) {
                error(file, lineNumber, e);
            } catch (Throwable t) {
                error(file, lineNumber, t);
                t.printStackTrace(out);
            }
        }


        int getLineNumber() {
            return lineNumber;
        }

        /**
         * Called when a file has been opened, before parsing begins.
         * This is always the first notification when reading a file.
         * This implementation does nothing.
         *
         * @param file the file
         */
        protected void startFile(Path file) { }

        /**
         * Called when the parser has finished reading a file.
         * This is always the last notification when reading a file,
         * unless any errors occur while closing the file.
         * This implementation does nothing.
         */
        protected void endFile() { }

        /**
         * Called when a doctype declaration is found, at the beginning of the file.
         * This implementation does nothing.
         * @param s the doctype declaration
         */
        protected void docType(String s) { }

        /**
         * Called when the opening tag of an HTML element is encountered.
         * This implementation does nothing.
         * @param name the name of the tag
         * @param attrs the attribute
         * @param selfClosing whether or not this is a self-closing tag
         */
        protected void startElement(String name, Map<String,String> attrs, boolean selfClosing) { }

        /**
         * Called when the closing tag of an HTML tag is encountered.
         * This implementation does nothing.
         * @param name the name of the tag
         */
        protected void endElement(String name) { }

        /**
         * Called when an error has been encountered.
         * @param file the file being read
         * @param lineNumber the line number of line containing the error
         * @param message a description of the error
         */
        protected void error(Path file, int lineNumber, String message) {
            out.println(file + ":" + lineNumber + ": " + message);
        }

        /**
         * Called when an exception has been encountered.
         * @param file the file being read
         * @param lineNumber the line number of the line being read when the exception was found
         * @param t the exception
         */
        protected void error(Path file, int lineNumber, Throwable t) {
            out.println(file + ":" + lineNumber + ": " + t);
        }

        private void nextChar() throws IOException {
            ch = in.read();
            if (ch == '\n')
                lineNumber++;
        }

        /**
         * Read the start or end of an HTML tag, or an HTML comment
         * {@literal <identifier attrs> } or {@literal </identifier> }
         * @throws java.io.IOException if there is a problem reading the file
         */
        private void html() throws IOException {
            nextChar();
            if (isIdentifierStart((char) ch)) {
                String name = readIdentifier().toLowerCase(Locale.US);
                Map<String,String> attrs = htmlAttrs();
                if (attrs != null) {
                    boolean selfClosing = false;
                    if (ch == '/') {
                        nextChar();
                        selfClosing = true;
                    }
                    if (ch == '>') {
                        nextChar();
                        startElement(name, attrs, selfClosing);
                        if (name.equals("script")) {
                            inScript = true;
                        }
                        return;
                    }
                }
            } else if (ch == '/') {
                nextChar();
                if (isIdentifierStart((char) ch)) {
                    String name = readIdentifier().toLowerCase(Locale.US);
                    skipWhitespace();
                    if (ch == '>') {
                        nextChar();
                        endElement(name);
                        if (name.equals("script")) {
                            inScript = false;
                        }
                        return;
                    }
                }
            } else if (ch == '!') {
                nextChar();
                if (ch == '-') {
                    nextChar();
                    if (ch == '-') {
                        nextChar();
                        while (ch != -1) {
                            int dash = 0;
                            while (ch == '-') {
                                dash++;
                                nextChar();
                            }
                            // Strictly speaking, a comment should not contain "--"
                            // so dash > 2 is an error, dash == 2 implies ch == '>'
                            // See http://www.w3.org/TR/html-markup/syntax.html#syntax-comments
                            // for more details.
                            if (dash >= 2 && ch == '>') {
                                nextChar();
                                return;
                            }

                            nextChar();
                        }
                    }
                } else if (ch == '[') {
                    nextChar();
                    if (ch == 'C') {
                        nextChar();
                        if (ch == 'D') {
                            nextChar();
                            if (ch == 'A') {
                                nextChar();
                                if (ch == 'T') {
                                    nextChar();
                                    if (ch == 'A') {
                                        nextChar();
                                        if (ch == '[') {
                                            while (true) {
                                                nextChar();
                                                if (ch == ']') {
                                                    nextChar();
                                                    if (ch == ']') {
                                                        nextChar();
                                                        if (ch == '>') {
                                                            nextChar();
                                                            return;
                                                        }
                                                    }
                                                }
                                            }

                                        }
                                    }
                                }
                            }
                        }
                    }
                } else {
                    StringBuilder sb = new StringBuilder();
                    while (ch != -1 && ch != '>') {
                        sb.append((char) ch);
                        nextChar();
                    }
                    Pattern p = Pattern.compile("(?is)doctype\\s+html\\s?.*");
                    String s = sb.toString();
                    if (p.matcher(s).matches()) {
                        docType(s);
                        return;
                    }
                }
            } else if (ch == '?') {
                nextChar();
                if (ch == 'x') {
                    nextChar();
                    if (ch == 'm') {
                        nextChar();
                        if (ch == 'l') {
                            Map<String,String> attrs = htmlAttrs();
                            if (ch == '?') {
                                nextChar();
                                if (ch == '>') {
                                    nextChar();
                                    xml = true;
                                    return;
                                }
                            }
                        }
                    }

                }
            }

            if (!inScript) {
                error(file, lineNumber, "bad html");
            }
        }

        /**
         * Read a series of HTML attributes, terminated by {@literal > }.
         * Each attribute is of the form {@literal identifier[=value] }.
         * "value" may be unquoted, single-quoted, or double-quoted.
         */
        private Map<String,String> htmlAttrs() throws IOException {
            Map<String, String> map = new LinkedHashMap<>();
            skipWhitespace();

            loop:
            while (isIdentifierStart((char) ch)) {
                String name = readAttributeName().toLowerCase(Locale.US);
                skipWhitespace();
                String value = null;
                if (ch == '=') {
                    nextChar();
                    skipWhitespace();
                    if (ch == '\'' || ch == '"') {
                        char quote = (char) ch;
                        nextChar();
                        StringBuilder sb = new StringBuilder();
                        while (ch != -1 && ch != quote) {
                            sb.append((char) ch);
                            nextChar();
                        }
                        value = sb.toString() // hack to replace common entities
                                .replace("&lt;", "<")
                                .replace("&gt;", ">")
                                .replace("&amp;", "&");
                        nextChar();
                    } else {
                        StringBuilder sb = new StringBuilder();
                        while (ch != -1 && !isUnquotedAttrValueTerminator((char) ch)) {
                            sb.append((char) ch);
                            nextChar();
                        }
                        value = sb.toString();
                    }
                    skipWhitespace();
                }
                map.put(name, value);
            }

            return map;
        }

        private boolean isIdentifierStart(char ch) {
            return Character.isUnicodeIdentifierStart(ch);
        }

        private String readIdentifier() throws IOException {
            StringBuilder sb = new StringBuilder();
            sb.append((char) ch);
            nextChar();
            while (ch != -1 && Character.isUnicodeIdentifierPart(ch)) {
                sb.append((char) ch);
                nextChar();
            }
            return sb.toString();
        }

        private String readAttributeName() throws IOException {
            StringBuilder sb = new StringBuilder();
            sb.append((char) ch);
            nextChar();
            while (ch != -1 && Character.isUnicodeIdentifierPart(ch)
                    || ch == '-'
                    || xml && ch == ':') {
                sb.append((char) ch);
                nextChar();
            }
            return sb.toString();
        }

        private boolean isWhitespace(char ch) {
            return Character.isWhitespace(ch);
        }

        private void skipWhitespace() throws IOException {
            while (isWhitespace((char) ch)) {
                nextChar();
            }
        }

        private boolean isUnquotedAttrValueTerminator(char ch) {
            switch (ch) {
                case '\f': case '\n': case '\r': case '\t':
                case ' ':
                case '"': case '\'': case '`':
                case '=': case '<': case '>':
                    return true;
                default:
                    return false;
            }
        }
    }

    /**
     * A class to check the links in a set of HTML files.
     */
    static class LinkChecker extends HtmlParser {
        private final Map<Path, IDTable> allFiles;
        private final Map<URI, IDTable> allURIs;

        private int files;
        private int links;
        private int badSchemes;
        private int duplicateIds;
        private int missingIds;

        private Path currFile;
        private IDTable currTable;
        private boolean html5;
        private boolean xml;

        private int errors;

        LinkChecker(PrintStream out, Function<Path,String> fileReader) {
            super(out, fileReader);
            allFiles = new HashMap<>();
            allURIs = new HashMap<>();
        }

        void checkDirectory(Path dir) throws IOException {
            checkFiles(List.of(dir), false, Collections.emptySet());
        }

        void checkFiles(List<Path> files, boolean skipSubdirs, Set<Path> excludeFiles) throws IOException {
            for (Path file : files) {
                Files.walkFileTree(file, new SimpleFileVisitor<Path>() {
                    int depth = 0;

                    @Override
                    public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
                        if ((skipSubdirs && depth > 0) || excludeFiles.contains(dir)) {
                            return FileVisitResult.SKIP_SUBTREE;
                        }
                        depth++;
                        return FileVisitResult.CONTINUE;
                    }

                    @Override
                    public FileVisitResult visitFile(Path p, BasicFileAttributes attrs) {
                        if (excludeFiles.contains(p)) {
                            return FileVisitResult.CONTINUE;
                        }

                        if (Files.isRegularFile(p) && p.getFileName().toString().endsWith(".html")) {
                            checkFile(p);
                        }
                        return FileVisitResult.CONTINUE;
                    }

                    @Override
                    public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
                        depth--;
                        return super.postVisitDirectory(dir, e);
                    }
                });
            }
        }

        void checkFile(Path file) {
            try {
                read(file);
            } catch (IOException e) {
                error(file, 0, e);
            }
        }

        int getErrorCount() {
            return errors;
        }

        public void report() {
            List<Path> missingFiles = getMissingFiles();
            if (!missingFiles.isEmpty()) {
                report("Missing files: (" + missingFiles.size() + ")");
                missingFiles.stream()
                        .sorted()
                        .forEach(this::reportMissingFile);

            }

            if (!allURIs.isEmpty()) {
                report(false, "External URLs:");
                allURIs.keySet().stream()
                        .sorted(new URIComparator())
                        .forEach(uri -> report(false, "  %s", uri.toString()));
            }

            int anchors = 0;
            for (IDTable t : allFiles.values()) {
                anchors += t.map.values().stream()
                        .filter(e -> !e.getReferences().isEmpty())
                        .count();
            }
            for (IDTable t : allURIs.values()) {
                anchors += t.map.values().stream()
                        .filter(e -> !e.references.isEmpty())
                        .count();
            }

            report(false, "Checked " + files + " files.");
            report(false, "Found " + links + " references to " + anchors + " anchors "
                    + "in " + allFiles.size() + " files and " + allURIs.size() + " other URIs.");
            report(!missingFiles.isEmpty(),   "%6d missing files", missingFiles.size());
            report(duplicateIds > 0, "%6d duplicate ids", duplicateIds);
            report(missingIds > 0,   "%6d missing ids", missingIds);

            Map<String, Integer> schemeCounts = new TreeMap<>();
            Map<String, Integer> hostCounts = new TreeMap<>(new HostComparator());
            for (URI uri : allURIs.keySet()) {
                String scheme = uri.getScheme();
                if (scheme != null) {
                    schemeCounts.put(scheme, schemeCounts.computeIfAbsent(scheme, s -> 0) + 1);
                }
                String host = uri.getHost();
                if (host != null) {
                    hostCounts.put(host, hostCounts.computeIfAbsent(host, h -> 0) + 1);
                }
            }

            if (schemeCounts.size() > 0) {
                report(false, "Schemes");
                schemeCounts.forEach((s, n) -> report(!isSchemeOK(s), "%6d %s", n, s));
            }

            if (hostCounts.size() > 0) {
                report(false, "Hosts");
                hostCounts.forEach((h, n) -> report(false, "%6d %s", n, h));
            }
        }

        private void report(String message, Object... args) {
            out.println(String.format(message, args));
        }

        private void report(boolean highlight, String message, Object... args) {
            out.print(highlight ? "* " : "  ");
            out.println(String.format(message, args));
        }

        private void reportMissingFile(Path file) {
            report("%s", relativePath(file));
            IDTable table = allFiles.get(file);
            Set<Path> refs = new TreeSet<>();
            for (ID id : table.map.values()) {
                if (id.references != null) {
                    for (Position p : id.references) {
                        refs.add(p.path);
                    }
                }
            }
            int n = 0;
            int MAX_REFS = 10;
            for (Path ref : refs) {
                report("    in " + relativePath(ref));
                if (++n == MAX_REFS) {
                    report("    ... and %d more", refs.size() - n);
                    break;
                }
            }
        }

        @Override
        public void startFile(Path path) {
            currFile = path.toAbsolutePath().normalize();
            currTable = allFiles.computeIfAbsent(currFile, p -> new IDTable(p));
            html5 = false;
            files++;
        }

        @Override
        public void endFile() {
            currTable.check();
        }

        @Override
        public void docType(String doctype) {
            html5 = doctype.matches("(?i)<\\?doctype\\s+html>");
        }

        @Override @SuppressWarnings("fallthrough")
        public void startElement(String name, Map<String, String> attrs, boolean selfClosing) {
            int line = getLineNumber();
            switch (name) {
                case "a":
                    String nameAttr = html5 ? null : attrs.get("name");
                    if (nameAttr != null) {
                        foundAnchor(line, nameAttr);
                    }
                    // fallthrough
                case "link":
                    String href = attrs.get("href");
                    if (href != null) {
                        foundReference(line, href);
                    }
                    break;
            }

            String idAttr = attrs.get("id");
            if (idAttr != null) {
                foundAnchor(line, idAttr);
            }
        }

        @Override
        public void endElement(String name) { }

        private void foundAnchor(int line, String name) {
            currTable.addID(line, name);
        }

        private void foundReference(int line, String ref) {
            links++;
            try {
                URI uri = new URI(ref);
                if (uri.isAbsolute()) {
                    foundReference(line, uri);
                } else {
                    Path p;
                    String uriPath = uri.getPath();
                    if (uriPath == null || uriPath.isEmpty()) {
                        p = currFile;
                    } else {
                        p = currFile.getParent().resolve(uriPath).normalize();
                    }
                    foundReference(line, p, uri.getFragment());
                }
            } catch (URISyntaxException e) {
                error(currFile, line, "invalid URI: " + e);
            }
        }

        private void foundReference(int line, Path p, String fragment) {
            IDTable t = allFiles.computeIfAbsent(p, key -> new IDTable(key));
            t.addReference(fragment, currFile, line);
        }

        private void foundReference(int line, URI uri) {
            if (!isSchemeOK(uri.getScheme())) {
                error(currFile, line, "bad scheme in URI");
                badSchemes++;
            }

            String fragment = uri.getFragment();
            try {
                URI noFrag = new URI(uri.toString().replaceAll("#\\Q" + fragment + "\\E$", ""));
                IDTable t = allURIs.computeIfAbsent(noFrag, key -> new IDTable(key.toString()));
                t.addReference(fragment, currFile, line);
            } catch (URISyntaxException e) {
                throw new Error(e);
            }
        }

        private boolean isSchemeOK(String uriScheme) {
            if (uriScheme == null) {
                return true;
            }

            switch (uriScheme) {
                case "file":
                case "ftp":
                case "http":
                case "https":
                case "javascript":
                case "mailto":
                    return true;

                default:
                    return false;
            }
        }

        private List<Path> getMissingFiles() {
            return allFiles.entrySet().stream()
                    .filter(e -> !Files.exists(e.getKey()))
                    .map(e -> e.getKey())
                    .collect(Collectors.toList());
        }

        @Override
        protected void error(Path file, int lineNumber, String message) {
            super.error(relativePath(file), lineNumber, message);
            errors++;
        }

        @Override
        protected void error(Path file, int lineNumber, Throwable t) {
            super.error(relativePath(file), lineNumber, t);
            errors++;
        }

        private Path relativePath(Path path) {
            return path.startsWith(currDir) ? currDir.relativize(path) : path;
        }

        /**
         * A position in a file, as identified by a file name and line number.
         */
        static class Position implements Comparable<Position> {
            Path path;
            int line;

            Position(Path path, int line) {
                this.path = path;
                this.line = line;
            }

            @Override
            public int compareTo(Position o) {
                int v = path.compareTo(o.path);
                return v != 0 ? v : Integer.compare(line, o.line);
            }

            @Override
            public boolean equals(Object obj) {
                if (this == obj) {
                    return true;
                } else if (obj == null || getClass() != obj.getClass()) {
                    return false;
                } else {
                    final Position other = (Position) obj;
                    return Objects.equals(this.path, other.path)
                            && this.line == other.line;
                }
            }

            @Override
            public int hashCode() {
                return Objects.hashCode(path) * 37 + line;
            }
        }

        /**
         * Infor for an ID within an HTML file, and a set of positions that reference it.
         */
        static class ID {
            boolean declared;
            Set<Position> references;

            Set<Position> getReferences() {
                return (references) == null ? Collections.emptySet() : references;
            }
        }

        /**
         * A table for the set of IDs in an HTML file.
         */
        class IDTable {
            private String name;
            private boolean checked;
            private final Map<String, ID> map = new HashMap<>();

            IDTable(Path p) {
                this(relativePath(p).toString());
            }

            IDTable(String name) {
                this.name = name;
            }

            void addID(int line, String name) {
                if (checked) {
                    throw new IllegalStateException("Adding ID after file has been read");
                }
                Objects.requireNonNull(name);
                ID id = map.computeIfAbsent(name, x -> new ID());
                if (id.declared) {
                    error(currFile, line, "name already declared: " + name);
                    duplicateIds++;
                } else {
                    id.declared = true;
                }
            }

            void addReference(String name, Path from, int line) {
                if (checked) {
                    if (name != null) {
                        ID id = map.get(name);
                        if (id == null || !id.declared) {
                            error(from, line, "id not found: " + this.name + "#" + name);
                        }
                    }
                } else {
                    ID id = map.computeIfAbsent(name, x -> new ID());
                    if (id.references == null) {
                        id.references = new TreeSet<>();
                    }
                    id.references.add(new Position(from, line));
                }
            }

            void check() {
                map.forEach((name, id) -> {
                    if (name != null && !id.declared) {
                        //log.error(currFile, 0, "id not declared: " + name);
                        for (Position ref : id.references) {
                            error(ref.path, ref.line, "id not found: " + this.name + "#" + name);
                        }
                        missingIds++;
                    }
                });
                checked = true;
            }
        }

        static class URIComparator implements Comparator<URI> {
            final HostComparator hostComparator = new HostComparator();

            @Override
            public int compare(URI o1, URI o2) {
                if (o1.isOpaque() || o2.isOpaque()) {
                    return o1.compareTo(o2);
                }
                String h1 = o1.getHost();
                String h2 = o2.getHost();
                String s1 = o1.getScheme();
                String s2 = o2.getScheme();
                if (h1 == null || h1.isEmpty() || s1 == null || s1.isEmpty()
                        || h2 == null || h2.isEmpty() || s2 == null || s2.isEmpty()) {
                    return o1.compareTo(o2);
                }
                int v = hostComparator.compare(h1, h2);
                if (v != 0) {
                    return v;
                }
                v = s1.compareTo(s2);
                if (v != 0) {
                    return v;
                }
                return o1.compareTo(o2);
            }
        }

        static class HostComparator implements Comparator<String> {
            @Override
            public int compare(String h1, String h2) {
                List<String> l1 = new ArrayList<>(Arrays.asList(h1.split("\\.")));
                Collections.reverse(l1);
                String r1 = String.join(".", l1);
                List<String> l2 = new ArrayList<>(Arrays.asList(h2.split("\\.")));
                Collections.reverse(l2);
                String r2 = String.join(".", l2);
                return r1.compareTo(r2);
            }
        }

    }
}