validating input files added

lib/common/ndcg.rb

@@ -0,0 +1,26 @@
+class NDCG
+
+	def initialize(list)
+		@list = list
+		@at = 10
+	end
+
+	def CG
+		@list[0..@at].inject(0) { |sum,x| sum + x}
+	end
+
+	def DCG(list)
+		list[0..@at].each.with_index.inject(0) { |sum,(x,i) | sum + ((2**x - 1) / (Math::log(i+2) / Math::log(2)))}
+	end
+
+	def IDCG
+		self.DCG(@list.sort.reverse)
+	end
+
+	def at num
+		@at = num - 1
+		dcg = DCG(@list).to_f
+		idcg = IDCG()
+		return (idcg == 0 ? 0 : (dcg / idcg))
+	end
+end

lib/evaluation/evaluate.rb

@@ -0,0 +1,22 @@
+require './lib/evaluation/ndcg'
+
+module EvaluateModule
+
+	def self.included(base)
+		base.send :include, InstanceMethods
+	end
+
+	module InstanceMethods
+		def initialize(solution_file, prediction_file)
+			@solution_file = solution_file
+			@prediction_file = prediction_file
+		end
+
+		def by(method)
+			case method
+			when "NDCG@10"
+				return NDCGev.calculate(@solution_file, @prediction_file, 10)
+			end
+		end
+	end
+end

lib/evaluation/ndcg.rb

@@ -0,0 +1,40 @@
+require './lib/common/ndcg.rb'
+
+class NDCGev
+
+	def self.calculate(real_solution, my_solution, at)
+		real = File.readlines(real_solution)
+		my = File.readlines(my_solution)
+
+		if(my.size != real.size)
+			abort("some predictions are missing")
+		end
+
+		index = 0
+		sum = 0
+		c = 0
+		while index < my.size
+			line = my[index]
+			real_list = []
+			temp_index = index
+			while temp_index < my.size
+				words = real[temp_index].split(',')
+				break if words[0] != my[index].split(',')[0]
+				real_list << [words[1],words[2].strip.to_i]
+				temp_index += 1
+			end
+
+			size = real_list.size
+
+			my_rel = my[index..temp_index-1].map { |l| real_list.find { |w| w[0] == l.split(',')[1]} [1]} 
+			ndcg = NDCG.new(my_rel).at(at)
+			if(ndcg != 0 or (ndcg == 0 and my_rel.uniq != [0]))
+				sum += ndcg
+				c += 1
+			end
+			index = temp_index
+		end
+
+		return sum.to_f / c
+	end
+end

lib/inits/config.rb

@@ -1,6 +1,6 @@
 require 'yaml'
 
-ERRORS = YAML::load(File.open("./lib/inits/errors.yml")).inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+$ERRORS = YAML::load(File.open("./lib/inits/errors.yml")).inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
 
 class Conf
 
@@ -12,9 +12,9 @@ def self.init(config_file)
 			self.validate @@params
 
 		rescue Psych::SyntaxError
-			abort(ERRORS[:syntax])
+			abort($ERRORS[:syntax])
 		rescue Errno::ENOENT
-			abort(ERRORS[:conf_not_found])
+			abort($ERRORS[:conf_not_found])
 		end
 	end
 
@@ -38,9 +38,19 @@ def self.get_output
 		return @@params[:out]
 	end
 
+	def self.get_ev_method
+		return @@params[:evaluation]["method"]
+	end
+
+	def self.get_ev_solution
+		return @@params[:evaluation]["solution_file"]
+	end
+
 	private
 		def self.validate params
-			abort(ERRORS[:no_out]) unless params[:out]
-			abort(ERRORS[:no_method]) unless params[:method]
+			abort($ERRORS[:no_out]) unless params[:out]
+			abort($ERRORS[:no_method]) unless params[:method]
+			#abort(ERRORS[:no_ev_method]) (if params[:evaluation] and !params[:evaluation]["method"])
+			abort($ERRORS[:no_ev_solution]) if (params[:evaluation] and !params[:evaluation]["solution_file"])
 		end
 end

lib/inits/config.yml

@@ -1,16 +1,31 @@
 ---
-method:	borda
+method: kemeny
 out: /home/pooya/Projects/UTR/samples/output.dat
 estimators:
-    listnet:
-      path: /home/pooya/Projects/UTR/samples/listnet.dat
-      weight: 2
-    ranknet:
-      path: /home/pooya/Projects/UTR/samples/ranknet.dat
-      weight: 3
-    listmle:
-      path: /home/pooya/Projects/UTR/samples/listmle.dat
-      weight: 3
-    svm:
-      path: /home/pooya/Projects/UTR/samples/svm.dat
+    pooya:
+      path: /home/pooya/Projects/UTR/samples/pooya.dat
       weight: 1
+    hamed:
+      path: /home/pooya/Projects/UTR/samples/hamed.dat
+      weight: 2
+    javid:
+      path: /home/pooya/Projects/UTR/samples/javid.dat
+      weight: 5
+
+# method:	kemeny
+# out: /home/pooya/Projects/UTR/samples/output.dat
+# estimators:
+#     listnet:
+#       path: /home/pooya/Projects/UTR/samples/listnet.dat
+#       weight: 2
+#     ranknet:
+#       path: /home/pooya/Projects/UTR/samples/ranknet.dat
+#       weight: 3
+#     listmle:
+#       path: /home/pooya/Projects/UTR/samples/listmle.dat
+#       weight: 3
+#     svm:
+#       path: /home/pooya/Projects/UTR/samples/svm.dat
+#       weight: 1
+# evaluation:
+#     solution_file: /home/pooya/Projects/UTR/samples/solution.dat

lib/inits/errors.yml

@@ -1,5 +1,9 @@
 ---
-syntax: Your YML config file has syntax error
+syntax: Your YML config file has syntax error.
 conf_not_found: Config file not found. maybe you should use absolute path.
 no_out: You haven't specified output file in the config file.
-no_method: You haven't specified aggregation method in the config file.
+no_method: You haven't specified aggregation method in the config file.
+no_ev_method: no evaluation method found in the config file.
+no_ev_solution: no evaluation solution path found in the config file.
+group_ids_not_match: group ids in ranking files do not match with each other.
+instance_ids_not_match: instance ids in ranking files do not match with each other.

lib/methods/kemeny.rb

@@ -27,8 +27,8 @@ def run data
 		end
 
 		def aggregate candidates
-			puts "candidates: #{candidates}"
-			puts ""
+			# puts "candidates: #{candidates}"
+			# puts ""
 
 			majority_table = Hash.new { |h,k| h[k] = {} }
 			(0..candidates.size - 1).each do |i| # initializing the majority table
@@ -56,8 +56,8 @@ def aggregate candidates
 			keys = candidates.keys
 			result = keys
 
-			puts "before: #{result}"
-			puts "majority table: #{majority_table}"
+			# puts "before: #{result}"
+			# puts "majority table: #{majority_table}"
 			comp = Proc.new do |value1, value2|
 				j = candidates.keys.index(value1)
 				jp = candidates.keys.index(value2)
@@ -72,7 +72,7 @@ def aggregate candidates
 
 			QuickSort.sort!(result, comp)
 
-			puts "result: #{result}"
+#			puts "result: #{result}"
 			return result
 		end
 	end

lib/utr.rb

@@ -1,6 +1,7 @@
 require './lib/inits/config.rb'
 require './lib/methods/kemeny'
 require './lib/methods/borda'
+require './lib/evaluation/evaluate'
 
 class UTR
 
@@ -18,6 +19,10 @@ class Borda
 		end
 	end
 
+	class Evaluatation
+		include EvaluateModule
+	end
+
 	def load_data
 		data = Hash.new { |h,k| h[k] = Hash.new { |hh,kk| hh[kk] = {} }}
 		Conf.estimators.each do |estimator|
@@ -27,6 +32,9 @@ def load_data
 				data[estimator][group_id][instance_id] = score
 			end
 		end
+
+		validate data
+
 		return data
 	end
 
@@ -50,5 +58,24 @@ def run
 			dump_result Methods::Borda.run data
 		end
 	end
+
+	def evaluate(method)
+		Conf.estimators.each do |estimator|
+			result = Evaluatation.new(Conf.get_ev_solution, Conf.get_path(estimator)).by(method)
+			puts "#{estimator} #{method} -> #{result}"
+		end
+		result = Evaluatation.new(Conf.get_ev_solution, Conf.get_output).by(method)
+		puts "#{Conf.method} aggregation method #{method} -> #{result} "
+	end
+
+	private
+	def validate data
+		data.keys[0..-2].each_with_index do |estimator, i|
+			abort(ERROR[:group_ids_not_match]) if data[estimator].keys.sort != data[data.keys[i+1]].keys.sort
+			data[estimator].each do |group_id, hash|
+				abort($ERRORS[:instance_ids_not_match]) if hash.keys.sort != data[data.keys[i+1]][group_id].keys.sort
+			end
+		end
+	end
 
 end

rank.rb

@@ -1,4 +1,5 @@
 require './lib/utr.rb'
 
 utr = UTR.new("/home/pooya/Projects/UTR/lib/inits/config.yml")
-utr.run
+utr.run
+#utr.evaluate("NDCG@10")

samples/hamed.dat

@@ -0,0 +1,9 @@
+1,8,9
+1,5,8
+1,4,7
+1,2,6
+1,7,5
+1,3,4
+1,9,3
+1,1,2
+1,6,1

samples/javid.dat

@@ -0,0 +1,9 @@
+1,8,9
+1,6,8
+1,1,7
+1,4,6
+1,9,5
+1,7,4
+1,5,3
+1,2,2
+1,3,1

samples/output.dat

@@ -1,10 +1,9 @@
-1,222,5
-1,333,4
-1,444,3
-1,111,2
-1,555,1
-2,236,5
-2,234,4
-2,233,3
-2,231,2
-2,232,1
+1,8,9
+1,6,8
+1,1,7
+1,4,6
+1,9,5
+1,7,4
+1,5,3
+1,2,2
+1,3,1

samples/pooya.dat

@@ -0,0 +1,9 @@
+1,5,9
+1,9,8
+1,8,7
+1,4,6
+1,2,5
+1,6,4
+1,3,3
+1,1,2
+1,7,1

samples/solution.dat

@@ -0,0 +1,10 @@
+1,333,5
+1,222,4
+1,555,3
+1,444,2
+1,111,1
+2,233,5
+2,234,4
+2,231,3
+2,232,2
+2,236,1

specs/kemeny_spec.rb

@@ -0,0 +1,4 @@
+describe Kemeny do
+	it "should rank correctly" do 
+	end
+end