Introduced multiclass LibLinear

shogun-toolbox · Mar 3, 2012 · bd93073 · bd93073
1 parent 995af1b
commit bd93073
Show file tree

Hide file tree

Showing 11 changed files with 268 additions and 42 deletions.
diff --git a/examples/undocumented/python_modular/classifier_multiclassliblinear_modular.py b/examples/undocumented/python_modular/classifier_multiclassliblinear_modular.py
@@ -0,0 +1,27 @@
+from tools.load import LoadMatrix
+lm=LoadMatrix()
+
+traindat = lm.load_numbers('../data/fm_train_real.dat')
+testdat = lm.load_numbers('../data/fm_test_real.dat')
+label_traindat = lm.load_labels('../data/label_train_multiclass.dat')
+
+parameter_list = [[traindat,testdat,label_traindat,2.1,1,1e-5],[traindat,testdat,label_traindat,2.2,1,1e-5]]
+
+def classifier_multiclassliblinear_modular (fm_train_real=traindat,fm_test_real=testdat,label_train_multiclass=label_traindat,width=2.1,C=1,epsilon=1e-5):
+	from shogun.Features import RealFeatures, Labels
+	from shogun.Classifier import MulticlassLibLinear
+
+	feats_train=RealFeatures(fm_train_real)
+	feats_test=RealFeatures(fm_test_real)
+
+	labels=Labels(label_train_multiclass)
+
+	classifier = MulticlassLibLinear(C,feats_train,labels)
+	classifier.train()
+
+	out = classifier.apply(feats_test).get_labels()
+	return out
+
+if __name__=='__main__':
+	print 'MulticlassLibLinear'
+	classifier_multiclassliblinear_modular(*parameter_list[0])
diff --git a/examples/undocumented/python_modular/classifier_multiclassmachine_modular.py b/examples/undocumented/python_modular/classifier_multiclassmachine_modular.py
@@ -0,0 +1,32 @@
+from tools.load import LoadMatrix
+lm=LoadMatrix()
+
+traindat = lm.load_numbers('../data/fm_train_real.dat')
+testdat = lm.load_numbers('../data/fm_test_real.dat')
+label_traindat = lm.load_labels('../data/label_train_multiclass.dat')
+
+parameter_list = [[traindat,testdat,label_traindat,2.1,1,1e-5],[traindat,testdat,label_traindat,2.2,1,1e-5]]
+
+def classifier_multiclassmachine_modular (fm_train_real=traindat,fm_test_real=testdat,label_train_multiclass=label_traindat,width=2.1,C=1,epsilon=1e-5):
+	from shogun.Features import RealFeatures, Labels
+	from shogun.Kernel import GaussianKernel
+	from shogun.Classifier import LibSVM, KernelMulticlassMachine, ONE_VS_REST_STRATEGY
+
+	feats_train=RealFeatures(fm_train_real)
+	feats_test=RealFeatures(fm_test_real)
+	kernel=GaussianKernel(feats_train, feats_train, width)
+
+	labels=Labels(label_train_multiclass)
+
+	classifier = LibSVM(C, kernel, labels)
+	classifier.set_epsilon(epsilon)
+	mc_classifier = KernelMulticlassMachine(ONE_VS_REST_STRATEGY,kernel,classifier,labels)
+	mc_classifier.train()
+
+	kernel.init(feats_train, feats_test)
+	out = mc_classifier.apply().get_labels()
+	return out
+
+if __name__=='__main__':
+	print 'MulticlassMachine'
+	classifier_multiclassmachine_modular(*parameter_list[0])
diff --git a/src/interfaces/modular/Classifier.i b/src/interfaces/modular/Classifier.i
@@ -58,6 +58,7 @@
 %rename(MKLMultiClass) CMKLMultiClass;
 %rename(VowpalWabbit) CVowpalWabbit;
 %rename(ConjugateIndex) CConjugateIndex;
+%rename(MulticlassLibLinear) CMulticlassLibLinear;
 #ifdef USE_SVMLIGHT
 %rename(SVMLight) CSVMLight;
 %rename(DomainAdaptationSVM) CDomainAdaptationSVM;
@@ -116,6 +117,7 @@
 %include <shogun/classifier/vw/VowpalWabbit.h>
 %include <shogun/classifier/svm/DomainAdaptationSVMLinear.h>
 %include <shogun/classifier/ConjugateIndex.h>
+%include <shogun/classifier/svm/MulticlassLibLinear.h>
 
 #ifdef USE_SVMLIGHT
 

diff --git a/src/interfaces/modular/Classifier_includes.i b/src/interfaces/modular/Classifier_includes.i
@@ -42,6 +42,7 @@
  #include <shogun/classifier/mkl/MKLMultiClass.h>
  #include <shogun/classifier/vw/VowpalWabbit.h>
  #include <shogun/classifier/ConjugateIndex.h>
+ #include <shogun/classifier/svm/MulticlassLibLinear.h>
 #ifdef USE_SVMLIGHT
  #include <shogun/classifier/svm/SVMLight.h>
  #include <shogun/classifier/svm/SVMLightOneClass.h>

diff --git a/src/shogun/classifier/svm/LibLinear.cpp b/src/shogun/classifier/svm/LibLinear.cpp
@@ -195,18 +195,6 @@ bool CLibLinear::train_machine(CFeatures* data)
 			solve_l1r_lr(&prob, epsilon*CMath::min(pos,neg)/prob.l, Cp, Cn);
 			break;
 		}
-		case MCSVM_CS:
-		{
-			SG_NOTIMPLEMENTED;
-			/* TODO...
-			model_->w=Malloc(double, n*nr_class);
-			for(i=0;i<nr_class;i++)
-				for(j=start[i];j<start[i]+count[i];j++)
-					sub_prob.y[j] = i;
-			Solver_MCSVM_CS Solver(&sub_prob, nr_class, weighted_C, param->eps);
-			Solver.Solve(model_->w);
-			*/
-		}
 		default:
 			SG_ERROR("Error: unknown solver_type\n");
 			break;

diff --git a/src/shogun/classifier/svm/LibLinear.h b/src/shogun/classifier/svm/LibLinear.h
@@ -33,8 +33,6 @@ namespace shogun
 		/// L2 regularized linear SVM with L1-loss using dual coordinate descent
 		// (default since this is the standard SVM)
 		L2R_L1LOSS_SVC_DUAL,
-		/// linear multi-class svm by Crammer and Singer
-		MCSVM_CS,
 		/// L1 regularized SVM with L2-loss using dual coordinate descent
 		L1R_L2LOSS_SVC,
 		/// L1 regularized logistic regression

diff --git a/src/shogun/classifier/svm/MulticlassLibLinear.cpp b/src/shogun/classifier/svm/MulticlassLibLinear.cpp
@@ -0,0 +1,61 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Written (W) 2012 Sergey Lisitsyn
+ * Copyright (C) 2012 Sergey Lisitsyn
+ */
+
+
+#include <shogun/classifier/svm/MulticlassLibLinear.h>
+#include <shogun/mathematics/Math.h>
+
+using namespace shogun;
+
+bool CMulticlassLibLinear::train_machine(CFeatures* data)
+{
+	if (data)
+		set_features((CDotFeatures*)data);
+
+	int32_t num_vectors = m_features->get_num_vectors();
+	int32_t num_classes = labels->get_num_classes();
+
+	problem mc_problem;
+	mc_problem.l = num_vectors;
+	mc_problem.n = m_features->get_dim_feature_space();
+	mc_problem.y = SG_MALLOC(int32_t, mc_problem.l);
+	for (int32_t i=0; i<num_vectors; i++)
+		mc_problem.y[i] = labels->get_int_label(i);
+
+	mc_problem.x = m_features;
+	mc_problem.use_bias = true;
+
+	float64_t* w = SG_MALLOC(float64_t, mc_problem.n*num_classes);
+	float64_t* C = SG_MALLOC(float64_t, num_vectors);
+	for (int32_t i=0; i<num_vectors; i++)
+		C[i] = m_C;
+
+	Solver_MCSVM_CS solver(&mc_problem,num_classes,C,m_epsilon,m_max_iter);
+	solver.Solve(w);
+
+	m_machines.destroy_vector();
+	m_machines = SGVector<CMachine*>(num_classes);
+	for (int32_t i=0; i<num_classes; i++)
+	{
+		CLinearMachine* machine = new CLinearMachine();
+		SGVector<float64_t> cw(mc_problem.n);
+		for (int32_t j=0; j<mc_problem.n; j++)
+			cw[j] = w[j*num_classes+i];
+		machine->set_w(cw);
+
+		m_machines[i] = machine;
+	}
+
+	SG_FREE(C);
+	SG_FREE(w);
+	SG_FREE(mc_problem.y);
+
+	return true;
+}
diff --git a/src/shogun/classifier/svm/MulticlassLibLinear.h b/src/shogun/classifier/svm/MulticlassLibLinear.h
@@ -0,0 +1,111 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Written (W) 2012 Sergey Lisitsyn
+ * Copyright (C) 2012 Sergey Lisitsyn
+ */
+
+#ifndef _MULTICLASSLIBLINEAR_H___
+#define _MULTICLASSLIBLINEAR_H___
+
+#include <shogun/lib/common.h>
+#include <shogun/features/DotFeatures.h>
+#include <shogun/classifier/svm/SVM_linear.h>
+#include <shogun/machine/LinearMulticlassMachine.h>
+
+namespace shogun
+{
+
+/** @brief multiclass LibLinear wrapper */
+class CMulticlassLibLinear : public CLinearMulticlassMachine
+{
+	public:
+		/** default constructor  */
+		CMulticlassLibLinear() : CLinearMulticlassMachine()
+		{
+		}
+
+		/** standard constructor
+		 * @param C C regularication constant value
+		 * @param features features 
+		 * @param labs labels
+		 */
+		CMulticlassLibLinear(float64_t C, CDotFeatures* features, CLabels* labs) :
+			CLinearMulticlassMachine(ONE_VS_REST_STRATEGY,features,NULL,labs), m_C(C)
+		{
+			set_epsilon(1e-2);
+			set_max_iter(10000);
+		}
+
+		/** destructor */
+		virtual ~CMulticlassLibLinear()
+		{
+		}
+
+		/** get name */
+		virtual const char* get_name() const 
+		{
+			return "MulticlassLibLinear";
+		}
+
+		/** set C 
+		 * @param C C value
+		 */
+		inline void set_C(int32_t C) 
+		{
+			ASSERT(C>0);
+			m_C = C;
+		}
+		/** get C
+		 * @return C value
+		 */
+		inline int32_t get_C() const { return m_C; }
+
+		/** set epsilon 
+		 * @param epsilon epsilon value
+		 */
+		inline void set_epsilon(float64_t epsilon) 
+		{ 
+			ASSERT(epsilon>0);
+			m_epsilon = epsilon; 
+		}
+		/** get epsilon
+		 * @return epsilon value
+		 */
+		inline float64_t get_epsilon() const { return m_epsilon; }
+
+		/** set max iter
+		 * @param max_iter max iter value
+		 */
+		inline void set_max_iter(int32_t max_iter) 
+		{
+			ASSERT(max_iter>0);
+			m_max_iter = max_iter;
+		}
+		/** get max iter
+		 * @return max iter value
+		 */
+		inline int32_t get_max_iter() const { return m_max_iter; }
+
+
+protected:
+
+		/** train machine */
+		virtual bool train_machine(CFeatures* data = NULL);
+
+protected:
+
+		/** regularization constant for each machine */
+		float64_t m_C;
+
+		/** tolerance */
+		float64_t m_epsilon;
+
+		/** max number of iterations */
+		int32_t m_max_iter;
+};
+}
+#endif
diff --git a/src/shogun/classifier/svm/SVM_linear.cpp b/src/shogun/classifier/svm/SVM_linear.cpp
@@ -342,8 +342,8 @@ void l2r_l2_svc_fun::subXTv(double *v, double *XTv)
 
 Solver_MCSVM_CS::Solver_MCSVM_CS(const problem *p, int n_class, double *weighted_C, double epsilon, int max_it)
 {
-	this->w_size = prob->n;
-	this->l = prob->l;
+	this->w_size = p->n;
+	this->l = p->l;
 	this->nr_class = n_class;
 	this->eps = epsilon;
 	this->max_iter = max_it;
@@ -457,17 +457,18 @@ void Solver_MCSVM_CS::Solve(double *w)
 				if(y_index[i] < active_size_i[i])
 					G[y_index[i]] = 0;
 
-				SG_SNOTIMPLEMENTED;
-				/* FIXME
-				feature_node *xi = prob->x[i];
-				while(xi->index!= -1)
+				void* feature_iter = prob->x->get_feature_iterator(i);
+				int32_t feature_index = 0;
+				float64_t feature_value = 0.0;
+				bool got_feature = prob->x->get_next_feature(feature_index,feature_value,feature_iter);
+				while(got_feature)
 				{
-					double *w_i = &w[(xi->index-1)*nr_class];
+					double *w_i = &w[feature_index*nr_class];
 					for(m=0;m<active_size_i[i];m++)
-						G[m] += w_i[alpha_index_i[m]]*(xi->value);
-					xi++;
+						G[m] += w_i[alpha_index_i[m]]*(feature_value);
+					got_feature = prob->x->get_next_feature(feature_index,feature_value,feature_iter);
 				}
-				*/
+				prob->x->free_feature_iterator(feature_iter);
 
 				double minG = CMath::INFTY;
 				double maxG = -CMath::INFTY;
@@ -535,15 +536,18 @@ void Solver_MCSVM_CS::Solve(double *w)
 					}
 				}
 
-				/* FIXME
-				xi = prob->x[i];
-				while(xi->index != -1)
+				feature_iter = prob->x->get_feature_iterator(i);
+				feature_index = 0;
+				feature_value = 0.0;
+				got_feature = prob->x->get_next_feature(feature_index,feature_value,feature_iter);
+				while(got_feature)
 				{
-					double *w_i = &w[(xi->index-1)*nr_class];
+					double *w_i = &w[feature_index*nr_class];
 					for(m=0;m<nz_d;m++)
-						w_i[d_ind[m]] += d_val[m]*xi->value;
-					xi++;
-				}*/
+						w_i[d_ind[m]] += d_val[m]*feature_value;
+					got_feature = prob->x->get_next_feature(feature_index,feature_value,feature_iter);
+				}
+				prob->x->free_feature_iterator(feature_iter);
 			}
 		}
 
@@ -592,15 +596,15 @@ void Solver_MCSVM_CS::Solve(double *w)
 	SG_SINFO("Objective value = %lf\n",v);
 	SG_SINFO("nSV = %d\n",nSV);
 
-	delete [] alpha;
-	delete [] alpha_new;
-	delete [] index;
-	delete [] QD;
-	delete [] d_ind;
-	delete [] d_val;
-	delete [] alpha_index;
-	delete [] y_index;
-	delete [] active_size_i;
+	SG_FREE(alpha);
+	SG_FREE(alpha_new);
+	SG_FREE(index);
+	SG_FREE(QD);
+	SG_FREE(d_ind);
+	SG_FREE(d_val);
+	SG_FREE(alpha_index);
+	SG_FREE(y_index);
+	SG_FREE(active_size_i);
 }
 
 //

diff --git a/src/shogun/machine/KernelMulticlassMachine.h b/src/shogun/machine/KernelMulticlassMachine.h
@@ -118,7 +118,7 @@ class CKernelMulticlassMachine : public CMulticlassMachine
 			return m_kernel->get_num_vec_rhs();
 		}
 
-	private:
+	protected:
 
 		CKernel* m_kernel;