Merge pull request #367 from karlnapf/master

A draft for training on fixed kernel matrices/data in general
shogun-toolbox · Feb 14, 2012 · 3649f3b · 3649f3b
2 parents 7cf941e + dd9768a
commit 3649f3b
Show file tree

Hide file tree

Showing 31 changed files with 953 additions and 93 deletions.
diff --git a/examples/undocumented/libshogun/Makefile b/examples/undocumented/libshogun/Makefile
@@ -23,6 +23,7 @@ TARGETS =	  basic_minimal classifier_libsvm classifier_minimal_svm \
 		  parameter_modsel_parameters \
 		  evaluation_cross_validation_classification \
 		  evaluation_cross_validation_regression \
+		  evaluation_cross_validation_locked_comparison \
 		  modelselection_parameter_combination_test \
 		  modelselection_model_selection_parameters_test \
 		  modelselection_parameter_tree \
@@ -65,6 +66,7 @@ TARGETS =	  basic_minimal classifier_libsvm classifier_minimal_svm \
 		  classifier_conjugateindex \
 		  classifier_gaussiannaivebayes \
 		  library_cover_tree \
+		  kernel_machine_train_locked \
 
 all: $(TARGETS)
 

diff --git a/examples/undocumented/libshogun/evaluation_cross_validation_classification.cpp b/examples/undocumented/libshogun/evaluation_cross_validation_classification.cpp
@@ -107,10 +107,17 @@ void test_cross_validation()
 	cross->set_num_runs(100);
 	cross->set_conf_int_alpha(0.05);
 
+	/* this is optional and speeds everything up since the kernel matrix is
+	 * precomputed. May not work though. */
+	svm->data_lock(features, labels);
+
 	/* actual evaluation */
 	CrossValidationResult result=cross->evaluate();
 	result.print_result();
 
+	/* see above */
+	svm->data_unlock();
+
 	/* clean up */
 	SG_UNREF(cross);
 	SG_UNREF(features);

diff --git a/examples/undocumented/libshogun/evaluation_cross_validation_locked_comparison.cpp b/examples/undocumented/libshogun/evaluation_cross_validation_locked_comparison.cpp
@@ -0,0 +1,150 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Written (W) 2012 Heiko Strathmann
+ * Copyright (C) 2012 Berlin Institute of Technology and Max-Planck-Society
+ */
+
+#include <shogun/base/init.h>
+#include <shogun/features/SimpleFeatures.h>
+#include <shogun/features/Labels.h>
+#include <shogun/kernel/GaussianKernel.h>
+#include <shogun/classifier/svm/LibSVM.h>
+#include <shogun/classifier/svm/SVMLight.h>
+#include <shogun/evaluation/CrossValidation.h>
+#include <shogun/evaluation/StratifiedCrossValidationSplitting.h>
+#include <shogun/evaluation/ContingencyTableEvaluation.h>
+#include <shogun/lib/Time.h>
+
+using namespace shogun;
+
+void print_message(FILE* target, const char* str)
+{
+	fprintf(target, "%s", str);
+}
+
+void test_cross_validation()
+{
+	/* data matrix dimensions */
+	index_t num_vectors=500;
+	index_t num_features=50;
+
+	/* data means -1, 1 in all components, std deviation of sigma */
+	SGVector<float64_t> mean_1(num_features);
+	SGVector<float64_t> mean_2(num_features);
+	CMath::fill_vector(mean_1.vector, mean_1.vlen, -1.0);
+	CMath::fill_vector(mean_2.vector, mean_2.vlen, 1.0);
+	float64_t sigma=1.5;
+
+	/* fill data matrix around mean */
+	SGMatrix<float64_t> train_dat(num_features, num_vectors);
+	for (index_t i=0; i<num_vectors; ++i)
+	{
+		for (index_t j=0; j<num_features; ++j)
+		{
+			float64_t mean=i<num_vectors/2 ? mean_1.vector[0] : mean_2.vector[0];
+			train_dat.matrix[i*num_features+j]=CMath::normal_random(mean, sigma);
+		}
+	}
+
+	/* training features */
+	CSimpleFeatures<float64_t>* features=
+			new CSimpleFeatures<float64_t>(train_dat);
+	SG_REF(features);
+
+	/* training labels +/- 1 for each cluster */
+	SGVector<float64_t> lab(num_vectors);
+	for (index_t i=0; i<num_vectors; ++i)
+		lab.vector[i]=i<num_vectors/2 ? -1.0 : 1.0;
+
+	CLabels* labels=new CLabels(lab);
+
+	/* gaussian kernel */
+	CGaussianKernel* kernel=new CGaussianKernel();
+	kernel->set_width(10);
+	kernel->init(features, features);
+
+	/* create svm via libsvm */
+	float64_t svm_C=1;
+	float64_t svm_eps=0.0001;
+	CSVM* svm=new CLibSVM(svm_C, kernel, labels);
+	svm->set_epsilon(svm_eps);
+
+	/* train and output the normal way */
+	SG_SPRINT("starting normal training\n");
+	svm->train(features);
+	CLabels* output=svm->apply(features);
+
+	/* evaluation criterion */
+	CContingencyTableEvaluation* eval_crit=
+			new CContingencyTableEvaluation(ACCURACY);
+
+	/* evaluate training error */
+	float64_t eval_result=eval_crit->evaluate(output, labels);
+	SG_SPRINT("training accuracy: %f\n", eval_result);
+	SG_UNREF(output);
+
+	/* assert that regression "works". this is not guaranteed to always work
+	 * but should be a really coarse check to see if everything is going
+	 * approx. right */
+	ASSERT(eval_result<2);
+
+	/* splitting strategy */
+	index_t n_folds=5;
+	CStratifiedCrossValidationSplitting* splitting=
+			new CStratifiedCrossValidationSplitting(labels, n_folds);
+
+	/* cross validation instance, 10 runs, 95% confidence interval */
+	CCrossValidation* cross=new CCrossValidation(svm, features, labels,
+			splitting, eval_crit);
+
+	cross->set_num_runs(20);
+	cross->set_conf_int_alpha(0.05);
+
+	/* actual evaluation without fixex kernel matrix */
+
+	index_t repetitions=1;
+	SG_SPRINT("unlocked x-val\n");
+	kernel->init(features, features);
+	for (index_t i=0; i<repetitions; ++i)
+	{
+		CTime time;
+		time.start();
+		cross->evaluate().print_result();
+		time.stop();
+		SG_SPRINT("%f sec\n", time.cur_time_diff());
+	}
+
+	/* actual evaluation with five kernel matrix (restore features first) */
+	svm->data_lock(features, labels);
+	SG_SPRINT("locked x-val\n");
+	for (index_t i=0; i<repetitions; ++i)
+	{
+		CTime time;
+		time.start();
+		cross->evaluate().print_result();
+		time.stop();
+		SG_SPRINT("%f sec\n", time.cur_time_diff());
+	}
+
+	/* clean up */
+	SG_UNREF(cross);
+	SG_UNREF(features);
+	mean_1.destroy_vector();
+	mean_2.destroy_vector();
+}
+
+int main(int argc, char **argv)
+{
+	init_shogun(&print_message, &print_message, &print_message);
+
+	test_cross_validation();
+
+	exit_shogun();
+
+	return 0;
+}
+
diff --git a/examples/undocumented/libshogun/evaluation_cross_validation_regression.cpp b/examples/undocumented/libshogun/evaluation_cross_validation_regression.cpp
@@ -94,11 +94,18 @@ void test_cross_validation()
 	cross->set_num_runs(100);
 	cross->set_conf_int_alpha(0.05);
 
+	/* this is optional and speeds everything up since the kernel matrix is
+	 * precomputed. May not work though.*/
+	krr->data_lock(features, labels);
+
 	/* actual evaluation */
 	CrossValidationResult result=cross->evaluate();
 	SG_SPRINT("cross_validation estimate:\n");
 	result.print_result();
 
+	/* see above */
+	krr->data_unlock();
+
 	/* same crude assertion as for above evaluation */
 	ASSERT(result.mean<2);
 

diff --git a/examples/undocumented/libshogun/kernel_machine_train_locked.cpp b/examples/undocumented/libshogun/kernel_machine_train_locked.cpp
@@ -0,0 +1,150 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Written (W) 2012 Heiko Strathmann
+ * Copyright (C) 2012 Berlin Institute of Technology and Max-Planck-Society
+ */
+
+#include <shogun/base/init.h>
+#include <shogun/features/SimpleFeatures.h>
+#include <shogun/features/Labels.h>
+#include <shogun/kernel/LinearKernel.h>
+#include <shogun/classifier/svm/LibSVM.h>
+#include <shogun/evaluation/ContingencyTableEvaluation.h>
+
+using namespace shogun;
+
+void print_message(FILE* target, const char* str)
+{
+	fprintf(target, "%s", str);
+}
+
+void test()
+{
+	/* data matrix dimensions */
+	index_t num_vectors=6;
+	index_t num_features=2;
+
+	/* data means -1, 1 in all components, std deviation of 3 */
+	SGVector<float64_t> mean_1(num_features);
+	SGVector<float64_t> mean_2(num_features);
+	CMath::fill_vector(mean_1.vector, mean_1.vlen, -10.0);
+	CMath::fill_vector(mean_2.vector, mean_2.vlen, 10.0);
+	float64_t sigma=0.5;
+
+	CMath::display_vector(mean_1.vector, mean_1.vlen, "mean 1");
+	CMath::display_vector(mean_2.vector, mean_2.vlen, "mean 2");
+
+	/* fill data matrix around mean */
+	SGMatrix<float64_t> train_dat(num_features, num_vectors);
+	for (index_t i=0; i<num_vectors; ++i)
+	{
+		for (index_t j=0; j<num_features; ++j)
+		{
+			float64_t mean=i<num_vectors/2 ? mean_1.vector[0] : mean_2.vector[0];
+			train_dat.matrix[i*num_features+j]=CMath::normal_random(mean, sigma);
+		}
+	}
+
+	CMath::display_matrix(train_dat.matrix, train_dat.num_rows, train_dat.num_cols, "training data");
+
+	/* training features */
+	CSimpleFeatures<float64_t>* features=
+			new CSimpleFeatures<float64_t>(train_dat);
+	SG_REF(features);
+
+	/* training labels +/- 1 for each cluster */
+	SGVector<float64_t> lab(num_vectors);
+	for (index_t i=0; i<num_vectors; ++i)
+		lab.vector[i]=i<num_vectors/2 ? -1.0 : 1.0;
+
+	CMath::display_vector(lab.vector, lab.vlen, "training labels");
+
+	CLabels* labels=new CLabels(lab);
+	SG_REF(labels);
+
+	/* evaluation instance */
+	CContingencyTableEvaluation* eval=new CContingencyTableEvaluation(ACCURACY);
+
+	/* kernel */
+	CKernel* kernel=new CLinearKernel();
+	kernel->init(features, features);
+
+	/* create svm via libsvm */
+	float64_t svm_C=10;
+	float64_t svm_eps=0.0001;
+	CLibSVM* svm=new CLibSVM(svm_C, kernel, labels);
+	svm->set_epsilon(svm_eps);
+
+	/* now train a few times on different subsets on data and assert that
+	 * results are correc (data linear separable) */
+
+	svm->data_lock(features, labels);
+
+	SGVector<index_t> indices(4);
+	indices.vector[0]=1;
+	indices.vector[1]=2;
+	indices.vector[2]=3;
+	indices.vector[3]=4;
+	CMath::display_vector(indices.vector, indices.vlen, "training indices");
+	svm->train_locked(indices);
+	CLabels* output=svm->apply();
+	ASSERT(eval->evaluate(output, labels)==1);
+	CMath::display_vector(output->get_labels().vector, output->get_num_labels(), "apply() output");
+	SG_UNREF(output);
+
+	SG_SPRINT("\n\n");
+	indices.destroy_vector();
+	indices=SGVector<index_t>(3);
+	indices.vector[0]=1;
+	indices.vector[1]=2;
+	indices.vector[2]=3;
+	CMath::display_vector(indices.vector, indices.vlen, "training indices");
+	output=svm->apply();
+	ASSERT(eval->evaluate(output, labels)==1);
+	CMath::display_vector(output->get_labels().vector, output->get_num_labels(), "apply() output");
+	SG_UNREF(output);
+
+	SG_SPRINT("\n\n");
+	indices.destroy_vector();
+	indices=SGVector<index_t>(4);
+	indices.range_fill();
+	CMath::display_vector(indices.vector, indices.vlen, "training indices");
+	svm->train_locked(indices);
+	output=svm->apply();
+	ASSERT(eval->evaluate(output, labels)==1);
+	CMath::display_vector(output->get_labels().vector, output->get_num_labels(), "apply() output");
+	SG_UNREF(output);
+
+	SG_SPRINT("normal train\n");
+	svm->data_unlock();
+	svm->train();
+	output=svm->apply();
+	ASSERT(eval->evaluate(output, labels)==1);
+	CMath::display_vector(output->get_labels().vector, output->get_num_labels(), "output");
+	SG_UNREF(output);
+
+	/* clean up */
+	SG_UNREF(svm);
+	SG_UNREF(features);
+	SG_UNREF(eval);
+	SG_UNREF(labels);
+	mean_1.destroy_vector();
+	mean_2.destroy_vector();
+	indices.destroy_vector();
+}
+
+int main(int argc, char **argv)
+{
+	init_shogun(&print_message, &print_message, &print_message);
+
+	test();
+
+	exit_shogun();
+
+	return 0;
+}
+
diff --git a/examples/undocumented/libshogun/modelselection_grid_search_kernel.cpp b/examples/undocumented/libshogun/modelselection_grid_search_kernel.cpp
@@ -141,7 +141,10 @@ int main(int argc, char **argv)
 	CGridSearchModelSelection* grid_search=new CGridSearchModelSelection(
 			param_tree, cross);
 
-	CParameterCombination* best_combination=grid_search->select_model();
+	bool lock_data=true;
+	bool print_state=true;
+	CParameterCombination* best_combination=grid_search->select_model(
+			print_state, lock_data);
 	SG_SPRINT("best parameter(s):\n");
 	best_combination->print_tree();
 

diff --git a/examples/undocumented/libshogun/modelselection_grid_search_krr.cpp b/examples/undocumented/libshogun/modelselection_grid_search_krr.cpp
@@ -128,7 +128,10 @@ void test_cross_validation()
 			param_tree, cross);
 
 	/* print current combination */
-	CParameterCombination* best_combination=grid_search->select_model(true);
+	bool print_state=true;
+	bool lock_data=true;
+	CParameterCombination* best_combination=grid_search->select_model(
+			print_state, lock_data);
 	SG_SPRINT("best parameter(s):\n");
 	best_combination->print_tree();
 
@@ -137,6 +140,7 @@ void test_cross_validation()
 	/* larger number of runs to have tighter confidence intervals */
 	cross->set_num_runs(10);
 	cross->set_conf_int_alpha(0.01);
+	krr->data_lock(features, labels);
 	CrossValidationResult result=cross->evaluate();
 	SG_SPRINT("result: ");
 	result.print_result();