TestDiffs.java

/* *******************************************************************
 * Copyright (c) 1999-2001 Xerox Corporation,
 *               2002 Palo Alto Research Center, Incorporated (PARC).
 * All rights reserved.
 * This program and the accompanying materials are made available
 * under the terms of the Eclipse Public License v 2.0
 * which accompanies this distribution and is available at
 * https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt
 *
 * Contributors:
 *     Xerox/PARC     initial implementation
 * ******************************************************************/

package org.aspectj.testing.util;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.ListIterator;

import org.aspectj.util.LangUtil;

/**
 * Calculated differences between two test runs
 * based on their output files
 * assuming that tests are logged with prefix [PASS|FAIL]
 * (as they are when using <tt>-traceTestsMin</tt> with the Harness).
 * @see org.aspectj.testing.drivers.Harness
 */
public class TestDiffs { // XXX pretty dumb implementation

    /** @param args expected, actual test result files */
    public static void main(String[] args) {
        if ((null == args) || (2 > args.length)) {
            System.err.println("java " + TestDiffs.class.getName() + " expectedFile actualFile {test}");
            return;
        }
        File expected  = new File(args[0]);
        File actual = new File(args[1]);

        TestDiffs result = compareResults(expected, actual);

        System.out.println("## Differences between test runs");
        print(System.out, result.added, "added");
        print(System.out, result.missing, "missing");
        print(System.out, result.fixed, "fixed");
        print(System.out, result.broken, "broken");

        System.out.println("## Summary");
        System.out.println(" # expected " + result.expected.size() + " tests: " + args[0] );
        System.out.println(" #   actual " +   result.actual.size() + " tests: " + args[1]);
        StringBuffer sb = new StringBuffer();
        append(sb, result.added, " added");
        append(sb, result.missing, " missing");
        append(sb, result.broken, " broken");
        append(sb, result.fixed, " fixed");
        append(sb, result.stillPassing, " still passing");
        append(sb, result.stillFailing, " still failing");
        System.out.println(" #    diffs: " + sb);
    }

    /**
     * @param expected the expected/old File results with Harness -traceTestsMin lines
     * @param actual the actual/new File results with Harness -traceTestsMin lines
     * @return TestDiffs null if error, valid otherwise
     */
    public static TestDiffs compareResults(File expected, File actual) {
		List<TestResult> exp = null;
		List<TestResult> act = null;
        File reading = expected;
        try {
            exp = TestDiffs.readTestResults(expected, expected.getPath());
            reading = actual;
            act = TestDiffs.readTestResults(actual, actual.getPath());

            Diffs tests = Diffs.makeDiffs("tests", exp, act, TestResult.BY_NAME);
            // remove missing/unexpected (removed, added) tests from results
            // otherwise, unexpected-[pass|fail] look like [fixes|broken]
            List<TestResult> expResults = trimByName(exp, tests.missing);
            List<TestResult> actResults = trimByName(act, tests.unexpected);

            Diffs results = Diffs.makeDiffs("results", expResults, actResults, TestResult.BY_PASSNAME);

            // broken tests show up in results as unexpected-fail or missing-pass
            //  fixed tests show up in results as unexpected-pass or missing-fail
            ArrayList<TestResult> broken = new ArrayList<TestResult>();
            ArrayList<TestResult> fixed = new ArrayList<TestResult>();
            split(results.unexpected, fixed, broken);

            return new TestDiffs(
                exp,
                act,
                tests.missing,
                tests.unexpected,
                broken,
                fixed);
        } catch (IOException e) {
            System.err.println("error reading " + reading);
            e.printStackTrace(System.err); // XXX
            return null;
        }
    }

    private static void append(StringBuffer sb, List<TestResult> list, String label) {
        if (!LangUtil.isEmpty(list)) {
            if (0 < sb.length()) {
                sb.append(" ");
            }
            sb.append(list.size() + label);
        }
    }

    private static void print(PrintStream out, List<TestResult> list, String label) {
        if ((null == out) || LangUtil.isEmpty(list)) {
            return;
        }
//        int i = 0;
        final String suffix = " " + label;
        final String LABEL = list.size() + suffix;
        out.println("## START " + LABEL);
		for (Object o : list) {
			TestResult result = (TestResult) o;
			out.println(" " + result.test + " ## " + suffix);
		}
        out.println("##   END " + LABEL);
    }

    /**
	 * Create ArrayList with input TestResult list
     * but without elements in trim list,
     * comparing based on test name only.
	 * @param input
	 * @param trim
	 * @return ArrayList with all input except those in trim (by name)
	 */
	private static List<TestResult> trimByName(List<TestResult> input, List<TestResult> trim) {
		List<TestResult> result = new ArrayList<TestResult>(input);
        if (!LangUtil.isEmpty(input) && !LangUtil.isEmpty(trim)) {
            for (ListIterator<TestResult> iter = result.listIterator(); iter.hasNext();) {
				TestResult inputItem = iter.next();
				for (Object o : trim) {
					TestResult trimItem = (TestResult) o;
					if (inputItem.test.equals(trimItem.test)) {
						iter.remove();
						break;
					}
				}
			}
        }
        return result;
	}


    /** split input List by whether the TestResult element passed or failed */
    private static void split(List<TestResult> input, List<TestResult> pass, List<TestResult> fail) {
		for (Object o : input) {
			TestResult result = (TestResult) o;
			if (result.pass) {
				pass.add(result);
			} else {
				fail.add(result);
			}
		}
    }

    /**
     * Read a file of test results,
     * defined as lines starting with [PASS|FAIL]
     * (produced by Harness option <tt>-traceTestsmin</tt>).
     * @return ArrayList of TestResult, one for every -traceTestsMin line in File
     */
    private static ArrayList<TestResult> readTestResults(File file, String config) throws IOException {
        LangUtil.throwIaxIfNull(file, "file");
        if (null == config) {
            config = file.getPath();
        }
        ArrayList<TestResult> result = new ArrayList<TestResult>();
        FileReader in = null;
        try {
            in = new FileReader(file);
            BufferedReader input = new BufferedReader(in);
            String line;
            // XXX handle stream interleaving more carefully
            // XXX clip trailing ()
            // XXX fix elision in test name rendering by -traceTestsMin?
            while (null != (line = input.readLine())) {
                boolean pass = line.startsWith("PASS");
                boolean fail = false;
                if (!pass) {
                    fail = line.startsWith("FAIL");
                }
                if (pass || fail) {
                    String test = line.substring(4).trim();
                    result.add(new TestResult(test, config, pass));
                }
            }
        } finally {
            if (null != in) {
                try { in.close(); }
                catch (IOException e) {} // ignore
            }
        }
        return result;
    }

    private static List<TestResult> safeList(List<TestResult> list) {
        return (null == list
            ? Collections.EMPTY_LIST
            : Collections.unmodifiableList(list));
    }

    /** List of TestResult results from expected run. */
    public final List<TestResult> expected;

    /** List of TestResult results from actual run. */
    public final List<TestResult> actual;

    /** List of TestResult tests disappeared from test suite between expected and actual runs. */
    public final List<TestResult> missing;

    /** List of TestResult tests added to test suite between expected and actual runs. */
    public final List<TestResult> added;

    /** List of TestResult tests in both runs, expected to pass but actually failed */
    public final List<TestResult> broken;

    /** List of TestResult tests in both runs, expected to fail but actually passed */
    public final List<TestResult> fixed;

    /** List of TestResult passed tests in expected run */
    public final List<TestResult> expectedPassed;

    /** List of TestResult failed tests in expected run */
    public final List<TestResult> expectedFailed;

    /** List of TestResult passed tests in actual run */
    public final List<TestResult> actualPassed;

    /** List of TestResult tests failed in actual run */
    public final List<TestResult> actualFailed;

    /** List of TestResult tests passed in both expected and actual run */
    public final List<TestResult> stillPassing;

    /** List of TestResult tests failed in both expected and actual run */
    public final List<TestResult> stillFailing;

    private TestDiffs(
        List<TestResult> expected,
        List<TestResult> actual,
        List<TestResult> missing,
        List<TestResult> added,
        List<TestResult> broken,
        List<TestResult> fixed) {
        this.expected = safeList(expected);
        this.actual = safeList(actual);
        this.missing = safeList(missing);
        this.added = safeList(added);
        this.broken = safeList(broken);
        this.fixed = safeList(fixed);
        // expected[Passed|Failed]
		List<TestResult> passed = new ArrayList<TestResult>();
		List<TestResult> failed = new ArrayList<TestResult>();
        split(this.expected, passed, failed);
        expectedPassed = safeList(passed);
        expectedFailed = safeList(failed);

        // actual[Passed|Failed]
        passed = new ArrayList<TestResult>();
        failed = new ArrayList<TestResult>();
        split(this.actual, passed, failed);
        actualPassed = safeList(passed);
        actualFailed = safeList(failed);

        // stillPassing: expected.passed w/o broken, missingPasses
		passed = new ArrayList<TestResult>(expectedPassed);
        passed = trimByName(passed, this.broken);
        ArrayList<TestResult> missingPasses = new ArrayList<TestResult>();
        ArrayList<TestResult> missingFails = new ArrayList<TestResult>();
        split(this.missing, missingPasses, missingFails);
        passed = trimByName(passed, missingPasses);
        stillPassing = safeList(passed);

        // stillFailing: expected.failed w/o fixed, missingFails
		failed = new ArrayList<TestResult>(expectedFailed);
        failed = trimByName(failed, this.fixed);
        failed = trimByName(failed, missingFails);
        stillFailing = safeList(failed);
    }

    /** results of a test */
    public static class TestResult {
        public static final Comparator BY_PASSNAME = new Comparator() {
            public int compare(Object o1, Object o2) {
                if (o1 == o2) {
                    return 0;
                }
                TestResult lhs = (TestResult) o1;
                TestResult rhs = (TestResult) o2;
                return (lhs.pass == rhs.pass
                    ? lhs.test.compareTo(rhs.test)
                    : (lhs.pass ? 1 : -1 ));
            }

            public boolean equals(Object lhs, Object rhs) {
                return (0 == compare(lhs, rhs));
            }
        };

        public static final Comparator BY_NAME = new Comparator() {
            public int compare(Object o1, Object o2) {
                if (o1 == o2) {
                    return 0;
                }
                TestResult lhs = (TestResult) o1;
                TestResult rhs = (TestResult) o2;
                return lhs.test.compareTo(rhs.test);
            }

            public boolean equals(Object lhs, Object rhs) {
                return (0 == compare(lhs, rhs));
            }
        };

        //private static final ArrayList TESTS = new ArrayList();
        public static final String FIELDSEP = "\t";

        public final String test;
        public final String config;
        public final boolean pass;
        private final String toString;

        public TestResult(String test, String config, boolean pass) {
            LangUtil.throwIaxIfNull(test, "test");
            LangUtil.throwIaxIfNull(test, "config");
            this.test = test;
            this.config = config;
            this.pass = pass;
            toString = (pass ? "PASS" : "FAIL") + FIELDSEP + test + FIELDSEP + config;

        }

        /** @return [PASS|FAIL]{FIELDSEP}test{FIELDSEP}config */
        public String toString() {
            return toString;
        }
    }
}