Algorithm Implementation/Statistics/Cohen's kappa

Implemented and tested as described at Wikipedia . This is just a first working draft without any optimization but it works just fine. Comments are for debug purposes only.

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class CohensKappa {

	public static double computeKappa(List<List<Integer>> scores) {
		if (scores.size() != 2) {
			throw new IllegalArgumentException("Only scores from two annotators are valid for Cohen's Kappa");
		}

		if (scores.get(0).size() != scores.get(1).size()) {
			throw new IllegalArgumentException("The scores from both annotators must have the same length");
		}

		Map<Integer, Map<Integer, Integer>> table = new HashMap<Integer, Map<Integer, Integer>>();

		int tableSize = 0;

		for (int i = 0; i < scores.get(0).size(); i++) {
			int first = scores.get(0).get(i);
			int second = scores.get(1).get(i);

			if (!table.containsKey(first)) {
				table.put(first, new HashMap<Integer, Integer>());
			}

			if (!table.get(first).containsKey(second)) {
				table.get(first).put(second, 0);
			}

			table.get(first).put(second, table.get(first).get(second) + 1);

			if (first > tableSize) {
				tableSize = first + 1;
			}

			if (second > tableSize) {
				tableSize = second + 1;
			}
		}

		boolean DEBUG = true;
		if (DEBUG) {
			System.out.println(table);
		}

		// sum the rows and cols
		Map<Integer, Integer> sumRows = new HashMap<Integer, Integer>();
		Map<Integer, Integer> sumCols = new HashMap<Integer, Integer>();

		for (Map.Entry<Integer, Map<Integer, Integer>> entry : table.entrySet()) {
			int rowNumber = entry.getKey();
			int sumRow = 0;

			for (Map.Entry<Integer, Integer> rowEntry : entry.getValue().entrySet()) {
				int colNumber = rowEntry.getKey();
				int value = rowEntry.getValue();

				sumRow += value;

				if (!sumCols.containsKey(colNumber)) {
					sumCols.put(colNumber, 0);
				}

				sumCols.put(colNumber, sumCols.get(colNumber) + value);
			}

			sumRows.put(rowNumber, sumRow);
		}

		// sum the total
		int sumTotal = 0;
		for (Integer rowSum : sumRows.values()) {
			sumTotal += rowSum;
		}

		if (DEBUG) {
			System.out.println("Row sums: " + sumRows);
			System.out.println("Col sums: " + sumCols);
			System.out.println("Total: " + sumTotal);
			System.out.println("table size: " + tableSize);
		}

		// sum diagonal
		int sumDiagonal = 0;
		for (int i = 1; i <= tableSize; i++) {
			int value = 0;
			if (table.containsKey(i) && table.get(i).containsKey(i)) {
				value = table.get(i).get(i);
			}

			sumDiagonal += value;
		}

		if (DEBUG) {
			System.out.println("Sum diagonal: " + sumDiagonal);
		}

		double p = (double) sumDiagonal / (double) sumTotal;

		if (DEBUG) {
			System.out.println("p: " + p);
		}

		double peSum = 0;
		for (int i = 1; i <= tableSize; i++) {
			if (sumRows.containsKey(i) && sumCols.containsKey(i)) {
				peSum += (double) sumRows.get(i) * (double) sumCols.get(i);
			}
		}

		if (DEBUG) {
			System.out.println("pesum: " + peSum);
		}

		double pe = peSum / (sumTotal * sumTotal);

		if (DEBUG) {
			System.out.println("pe: " + pe);
		}

		return (p - pe) / (1.0d - pe);
	}

	public static void main(String[] args) {
		exampleWikipediaOne();
		exampleWikipediaTwo();
		examplePaper1();
		examplePaper2();
	}

	public static void exampleWikipediaOne() {
		List<Integer> one = new ArrayList<Integer>();
		List<Integer> two = new ArrayList<Integer>();
		for (int i = 0; i < 45; i++) {
			one.add(1);
			two.add(1);
		}

		for (int i = 0; i < 15; i++) {
			one.add(1);
			two.add(2);
		}

		for (int i = 0; i < 25; i++) {
			one.add(2);
			two.add(1);
		}

		for (int i = 0; i < 15; i++) {
			one.add(2);
			two.add(2);
		}

		List<List<Integer>> list = new ArrayList<List<Integer>>();
		list.add(one);
		list.add(two);

		System.out.println(computeKappa(list));
	}

	public static void exampleWikipediaTwo() {
		List<Integer> one = new ArrayList<Integer>();
		List<Integer> two = new ArrayList<Integer>();
		for (int i = 0; i < 25; i++) {
			one.add(1);
			two.add(1);
		}

		for (int i = 0; i < 35; i++) {
			one.add(1);
			two.add(2);
		}

		for (int i = 0; i < 5; i++) {
			one.add(2);
			two.add(1);
		}

		for (int i = 0; i < 35; i++) {
			one.add(2);
			two.add(2);
		}

		List<List<Integer>> list = new ArrayList<List<Integer>>();
		list.add(one);
		list.add(two);

		System.out.println(computeKappa(list));
	}

	/*
	 * Example from http://www-users.york.ac.uk/~mb55/msc/clinimet/week4/kappash2.pdf Table 4, pg. 3
	 */
	public static void examplePaper1() {
		List<Integer> one = new ArrayList<Integer>();
		List<Integer> two = new ArrayList<Integer>();

		for (int i = 0; i < 12; i++) {
			one.add(1);
			two.add(1);
		}

		for (int i = 0; i < 4; i++) {
			one.add(1);
			two.add(2);
		}

		for (int i = 0; i < 2; i++) {
			one.add(1);
			two.add(3);
		}

		for (int i = 0; i < 12; i++) {
			one.add(2);
			two.add(1);
		}

		for (int i = 0; i < 56; i++) {
			one.add(2);
			two.add(2);
		}

		for (int i = 0; i < 3; i++) {
			one.add(3);
			two.add(1);
		}

		for (int i = 0; i < 4; i++) {
			one.add(3);
			two.add(2);
		}

		for (int i = 0; i < 1; i++) {
			one.add(3);
			two.add(3);
		}

		List<List<Integer>> list = new ArrayList<List<Integer>>();
		list.add(one);
		list.add(two);

		System.out.println(computeKappa(list));
	}

	/*
     * Example from http://www-users.york.ac.uk/~mb55/msc/clinimet/week4/kappash2.pdf Table 6, pg. 4
	 */
	public static void examplePaper2() {
		List<Integer> one = new ArrayList<Integer>();
		List<Integer> two = new ArrayList<Integer>();

		for (int i = 0; i < 2; i++) {
			one.add(1);
			two.add(1);
		}

		for (int i = 0; i < 12; i++) {
			one.add(1);
			two.add(2);
		}

		for (int i = 0; i < 8; i++) {
			one.add(1);
			two.add(3);
		}

		for (int i = 0; i < 9; i++) {
			one.add(2);
			two.add(1);
		}

		for (int i = 0; i < 35; i++) {
			one.add(2);
			two.add(2);
		}

		for (int i = 0; i < 43; i++) {
			one.add(2);
			two.add(3);
		}

		for (int i = 0; i < 7; i++) {
			one.add(2);
			two.add(4);
		}

		for (int i = 0; i < 4; i++) {
			one.add(3);
			two.add(1);
		}

		for (int i = 0; i < 36; i++) {
			one.add(3);
			two.add(2);
		}

		for (int i = 0; i < 103; i++) {
			one.add(3);
			two.add(3);
		}

		for (int i = 0; i < 40; i++) {
			one.add(3);
			two.add(4);
		}

		for (int i = 0; i < 1; i++) {
			one.add(4);
			two.add(1);
		}

		for (int i = 0; i < 8; i++) {
			one.add(4);
			two.add(2);
		}

		for (int i = 0; i < 36; i++) {
			one.add(4);
			two.add(3);
		}

		for (int i = 0; i < 22; i++) {
			one.add(4);
			two.add(4);
		}

		List<List<Integer>> list = new ArrayList<List<Integer>>();
		list.add(one);
		list.add(two);

		System.out.println(computeKappa(list));
	}
}