Skip to content

Commit

Permalink
Introduced L1 logistic regression
Browse files Browse the repository at this point in the history
  • Loading branch information
@lisitsyn
lisitsyn committed Jul 17, 2012
1 parent f2bcaa9 commit 53e4cff
Show file tree
Hide file tree
Showing 6 changed files with 227 additions and 5 deletions.
2 changes: 2 additions & 0 deletions src/interfaces/modular/Classifier.i
View file
Expand Up @@ -63,6 +63,7 @@
%rename(SVMLightOneClass) CSVMLightOneClass;
#endif //USE_SVMLIGHT
%rename(FeatureBlockLogisticRegression) CFeatureBlockLogisticRegression;
%rename(L1LogisticRegression) CL1LogisticRegression;
%rename(DirectorLinearMachine) CDirectorLinearMachine;
%rename(DirectorKernelMachine) CDirectorKernelMachine;

Expand Down Expand Up @@ -108,6 +109,7 @@
%include <shogun/classifier/svm/NewtonSVM.h>
%include <shogun/machine/SLEPMachine.h>
%include <shogun/classifier/FeatureBlockLogisticRegression.h>
%include <shogun/classifier/L1LogisticRegression.h>
%include <shogun/machine/DirectorLinearMachine.h>
%include <shogun/machine/DirectorKernelMachine.h>

Expand Down
1 change: 1 addition & 0 deletions src/interfaces/modular/Classifier_includes.i
View file
Expand Up @@ -38,6 +38,7 @@
#endif //USE_SVMLIGHT

#include <shogun/classifier/FeatureBlockLogisticRegression.h>
#include <shogun/classifier/L1LogisticRegression.h>
#include <shogun/machine/DirectorLinearMachine.h>
#include <shogun/machine/DirectorKernelMachine.h>
%}
97 changes: 97 additions & 0 deletions src/shogun/classifier/L1LogisticRegression.cpp
View file
@@ -0,0 +1,97 @@
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* Copyright (C) 2012 Sergey Lisitsyn
*/

#include <shogun/classifier/L1LogisticRegression.h>
#include <shogun/lib/slep/slep_logistic.h>
#include <shogun/lib/slep/slep_options.h>

namespace shogun
{

CL1LogisticRegression::CL1LogisticRegression() :
CSLEPMachine()
{
}

CL1LogisticRegression::CL1LogisticRegression(
float64_t z, CDotFeatures* train_features,
CBinaryLabels* train_labels) :
CSLEPMachine(z,train_features,(CLabels*)train_labels)
{
}

CL1LogisticRegression::~CL1LogisticRegression()
{
}

bool CL1LogisticRegression::train_machine(CFeatures* data)
{
if (data && (CDotFeatures*)data)
set_features((CDotFeatures*)data);

ASSERT(features);
ASSERT(m_labels);

int32_t n_vecs = m_labels->get_num_labels();
SGVector<float64_t> y(n_vecs);
for (int32_t i=0; i<n_vecs; i++)
y[i] = ((CBinaryLabels*)m_labels)->get_label(i);

slep_options options = slep_options::default_options();
options.mode = PLAIN;
options.regularization = m_regularization;
options.termination = m_termination;
options.tolerance = m_tolerance;
options.max_iter = m_max_iter;
options.rsL2 = 0.0;

slep_result_t result = slep_logistic(features, y.vector, m_z, options);

int32_t n_feats = features->get_dim_feature_space();
SGVector<float64_t> new_w(n_feats);
for (int i=0; i<n_feats; i++)
new_w[i] = result.w[i];

set_bias(result.c[0]);

w = new_w;

return true;
}

float64_t CL1LogisticRegression::apply_one(int32_t vec_idx)
{
return CMath::exp(-(features->dense_dot(vec_idx, w.vector, w.vlen) + bias));
}

SGVector<float64_t> CL1LogisticRegression::apply_get_outputs(CFeatures* data)
{
if (data)
{
if (!data->has_property(FP_DOT))
SG_ERROR("Specified features are not of type CDotFeatures\n");

set_features((CDotFeatures*) data);
}

if (!features)
return SGVector<float64_t>();

int32_t num=features->get_num_vectors();
ASSERT(num>0);
ASSERT(w.vlen==features->get_dim_feature_space());

float64_t* out=SG_MALLOC(float64_t, num);
features->dense_dot_range(out, 0, num, NULL, w.vector, w.vlen, bias);
for (int32_t i=0; i<num; i++)
out[i] = 2.0/(1.0+CMath::exp(-out[i])) - 1.0;//*CMath::exp(-CMath::sign(out[i])*out[i]);
return SGVector<float64_t>(out,num);
}

}
58 changes: 58 additions & 0 deletions src/shogun/classifier/L1LogisticRegression.h
View file
@@ -0,0 +1,58 @@
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* Copyright (C) 2012 Sergey Lisitsyn
*/

#ifndef L1LOGISTICREGRESSION_H_
#define L1LOGISTICREGRESSION_H_

#include <shogun/lib/config.h>
#include <shogun/machine/SLEPMachine.h>

namespace shogun
{
/** @brief */
class CL1LogisticRegression : public CSLEPMachine
{

public:
MACHINE_PROBLEM_TYPE(PT_BINARY)

/** default constructor */
CL1LogisticRegression();

/** constructor
*
* @param z regularization coefficient
* @param training_data training features
* @param training_labels training labels
*/
CL1LogisticRegression(
float64_t z, CDotFeatures* training_data,
CBinaryLabels* training_labels);

/** destructor */
virtual ~CL1LogisticRegression();

/** get name */
virtual const char* get_name() const
{
return "L1LogisticRegression";
}

virtual float64_t apply_one(int32_t vec_idx);

protected:

virtual SGVector<float64_t> apply_get_outputs(CFeatures* data);

/** train machine */
virtual bool train_machine(CFeatures* data=NULL);

};
}
#endif
69 changes: 65 additions & 4 deletions src/shogun/lib/slep/slep_logistic.cpp
View file
Expand Up @@ -19,8 +19,8 @@
namespace shogun
{

double compute_regularizer(double* w, int n_vecs, int n_feats,
int n_blocks, const slep_options& options)
double compute_regularizer_logistic(double* w, int n_vecs, int n_feats,
int n_blocks, const slep_options& options)
{
double regularizer = 0.0;
switch (options.mode)
Expand Down Expand Up @@ -73,6 +73,11 @@ double compute_regularizer(double* w, int n_vecs, int n_feats,
regularizer = treeNorm(w, 1, n_feats, options.ind_t, options.n_nodes);
}
break;
case PLAIN:
{
for (int i=0; i<n_feats; i++)
regularizer += CMath::abs(w[i]);
}
default:
SG_SERROR("WHOA?\n");
}
Expand Down Expand Up @@ -256,6 +261,39 @@ double compute_lambda_logistic(
SG_FREE(ATb);
}
break;
case PLAIN:
{
double* ATb = SG_CALLOC(double, n_feats);

int m1 = 0;
int m2 = 0;
double b = 0.0;
for (int i=0; i<n_vecs; i++)
{
if (y[i]>0)
m1++;
else
m2++;
}

for (int i=0; i<n_vecs; i++)
{
if (y[i]>0)
b = double(m2) / (n_vecs*n_vecs);
else
b = -double(m1) / (n_vecs*n_vecs);

features->add_to_dense_vec(b,i,ATb,n_feats);
}
double max = 0.0;
for (int i=0; i<n_feats; i++)
{
if (CMath::abs(ATb[i]) > max)
max = CMath::abs(ATb[i]);
}
lambda_max = max;
}
break;
default:
SG_SERROR("WHOAA!\n");
}
Expand Down Expand Up @@ -287,6 +325,7 @@ slep_result_t slep_logistic(
break;
case FEATURE_GROUP:
case FEATURE_TREE:
case PLAIN:
n_tasks = 1;
n_blocks = options.n_feature_blocks;
break;
Expand All @@ -296,8 +335,13 @@ slep_result_t slep_logistic(
bool done = false;
bool gradient_break = false;

double rsL2 = options.rsL2;

if (options.regularization!=0)
{
lambda = compute_lambda_logistic(z, features, y, n_vecs, n_feats, n_blocks, options);
rsL2*= lambda;
}
else
lambda = z;

Expand Down Expand Up @@ -342,6 +386,7 @@ slep_result_t slep_logistic(
break;
case FEATURE_GROUP:
case FEATURE_TREE:
case PLAIN:
{
for (i=0; i<n_vecs; i++)
Aw[i] = features->dense_dot(i,w.matrix,n_feats);
Expand Down Expand Up @@ -421,6 +466,7 @@ slep_result_t slep_logistic(
break;
case FEATURE_GROUP:
case FEATURE_TREE:
case PLAIN:
{
gc[0] = 0.0;

Expand All @@ -443,6 +489,11 @@ slep_result_t slep_logistic(
//SG_SDEBUG("G=%f\n", SGVector<float64_t>::dot(g,g,n_feats*n_tasks));

fun_s /= n_vecs;

if (options.mode==PLAIN)
{
fun_s += rsL2/2 * SGVector<float64_t>::dot(w.matrix,w.matrix,n_feats);
}

for (i=0; i<n_feats*n_tasks; i++)
wp[i] = w[i];
Expand Down Expand Up @@ -490,6 +541,11 @@ slep_result_t slep_logistic(
altra(w.matrix, v, n_feats, options.ind_t, options.n_nodes, lambda/L);
}
break;
case PLAIN:
{
for (i=0; i<n_feats; i++)
w[i] = CMath::sign(v[i])*CMath::max(0.0,CMath::abs(v[i])-lambda/L);
}
default:
SG_SERROR("WHOA!!!\n");
}
Expand Down Expand Up @@ -521,20 +577,25 @@ slep_result_t slep_logistic(
break;
case FEATURE_GROUP:
case FEATURE_TREE:
case PLAIN:
{
for (i=0; i<n_vecs; i++)
{
Aw[i] = features->dense_dot(i, w.matrix, n_feats);
double aa = -y[i]*(Aw[i]+c[0]);
double bb = CMath::max(aa,0.0);

fun_x += (CMath::log(CMath::exp(-bb) + CMath::exp(aa-bb)) + bb)/n_vecs;
fun_x += (CMath::log(CMath::exp(-bb) + CMath::exp(aa-bb)) + bb);
}
fun_x /= n_vecs;
}
break;
default:
SG_SERROR("WHOAAA!!\n");
}

if (options.mode==PLAIN)
fun_x += rsL2/2 * SGVector<float64_t>::dot(w.matrix,w.matrix,n_feats);

double r_sum = SGVector<float64_t>::dot(v,v,n_feats*n_tasks);
double l_sum = fun_x - fun_s - SGVector<float64_t>::dot(v,g,n_feats*n_tasks);
Expand Down Expand Up @@ -568,7 +629,7 @@ slep_result_t slep_logistic(
for (t=0; t<n_tasks; t++)
ccp[t] = c[t] - cp[t];

double regularizer = compute_regularizer(w.matrix, n_vecs, n_feats, n_blocks, options);
double regularizer = compute_regularizer_logistic(w.matrix, n_vecs, n_feats, n_blocks, options);

funcp = func;
func = fun_x + lambda*regularizer;
Expand Down
5 changes: 4 additions & 1 deletion src/shogun/lib/slep/slep_options.h
View file
Expand Up @@ -26,7 +26,8 @@ IGNORE_IN_CLASSLIST enum slep_mode
MULTITASK_GROUP,
MULTITASK_TREE,
FEATURE_GROUP,
FEATURE_TREE
FEATURE_TREE,
PLAIN
};

IGNORE_IN_CLASSLIST struct slep_options
Expand All @@ -41,6 +42,7 @@ IGNORE_IN_CLASSLIST struct slep_options
int regularization;
int n_feature_blocks;
int* ind;
double rsL2;
double* ind_t;
double* G;
double* gWeight;
Expand All @@ -61,6 +63,7 @@ IGNORE_IN_CLASSLIST struct slep_options
opts.ind = NULL;
opts.ind_t = NULL;
opts.G = NULL;
opts.rsL2 = 0.0;
opts.mode = MULTITASK_GROUP;
return opts;
}
Expand Down

0 comments on commit 53e4cff

Please sign in to comment.