Skip to content

Commit

Permalink
Fixes for binary evaluations
Browse files Browse the repository at this point in the history
  • Loading branch information
@lisitsyn
lisitsyn committed May 21, 2012
1 parent 8d4b65b commit 17f015e
Show file tree
Hide file tree
Showing 9 changed files with 30 additions and 215 deletions.
2 changes: 1 addition & 1 deletion src/shogun/evaluation/BinaryClassEvaluation.h
View file
Expand Up @@ -12,7 +12,7 @@
#define BINARYCLASSEVALUATION_H_

#include <shogun/evaluation/Evaluation.h>
#include <shogun/labels/Labels.h>
#include <shogun/labels/BinaryLabels.h>

namespace shogun
{
Expand Down
12 changes: 7 additions & 5 deletions src/shogun/evaluation/ContingencyTableEvaluation.cpp
View file
Expand Up @@ -15,7 +15,9 @@ using namespace shogun;

float64_t CContingencyTableEvaluation::evaluate(CLabels* predicted, CLabels* ground_truth)
{
compute_scores(predicted,ground_truth);
ASSERT(predicted->get_label_type()==LT_BINARY);
ASSERT(ground_truth->get_label_type()==LT_BINARY);
compute_scores((CBinaryLabels*)predicted,(CBinaryLabels*)ground_truth);
switch (m_type)
{
case ACCURACY:
Expand Down Expand Up @@ -71,7 +73,7 @@ inline EEvaluationDirection CContingencyTableEvaluation::get_evaluation_directio
return ED_MINIMIZE;
}

void CContingencyTableEvaluation::compute_scores(CLabels* predicted, CLabels* ground_truth)
void CContingencyTableEvaluation::compute_scores(CBinaryLabels* predicted, CBinaryLabels* ground_truth)
{
ASSERT(ground_truth->get_label_type() == LT_BINARY);
ASSERT(predicted->get_label_type() == LT_BINARY);
Expand All @@ -85,16 +87,16 @@ void CContingencyTableEvaluation::compute_scores(CLabels* predicted, CLabels* gr

for (int i=0; i<predicted->get_num_labels(); i++)
{
if (((CBinaryLabels*) ground_truth)->get_label(i)==1)
if (ground_truth->get_label(i)==1)
{
if (CMath::sign(((CBinaryLabels*) predicted)->get_label(i))==1)
if (predicted->get_confidence(i)>=1)
m_TP += 1.0;
else
m_FN += 1.0;
}
else
{
if (CMath::sign(((CBinaryLabels*) predicted)->get_label(i))==1)
if (predicted->get_confidence(i)>=1)
m_FP += 1.0;
else
m_TN += 1.0;
Expand Down
185 changes: 1 addition & 184 deletions src/shogun/evaluation/ContingencyTableEvaluation.h
View file
Expand Up @@ -201,7 +201,7 @@ class CContingencyTableEvaluation: public CBinaryClassEvaluation
protected:

/** get scores for TP, FP, TN, FN */
void compute_scores(CLabels* predicted, CLabels* ground_truth);
void compute_scores(CBinaryLabels* predicted, CBinaryLabels* ground_truth);

/** type of measure to evaluate */
EContingencyTableMeasureType m_type;
Expand All @@ -224,188 +224,5 @@ class CContingencyTableEvaluation: public CBinaryClassEvaluation
/** number of false negative examples */
float64_t m_FN;
};

/** @brief class AccuracyMeasure
* used to measure accuracy of 2-class classifier.
*
* This class is also capable of measuring
* any other rate using get_[measure name] methods
* of CContingencyTableEvaluation class.
*
* Note that evaluate() should be called first.
*/
class CAccuracyMeasure: public CContingencyTableEvaluation
{
public:
/* constructor */
CAccuracyMeasure() : CContingencyTableEvaluation(ACCURACY) {};
/* virtual destructor */
virtual ~CAccuracyMeasure() {};
/* name */
virtual inline const char* get_name() const { return "AccuracyMeasure"; };
};

/** @brief class ErrorRateMeasure
* used to measure error rate of 2-class classifier.
*
* This class is also capable of measuring
* any other rate using get_[measure name] methods
* of CContingencyTableEvaluation class.
*
* Note that evaluate() should be called first.
*/
class CErrorRateMeasure: public CContingencyTableEvaluation
{
public:
/* constructor */
CErrorRateMeasure() : CContingencyTableEvaluation(ERROR_RATE) {};
/* virtual destructor */
virtual ~CErrorRateMeasure() {};
/* name */
virtual inline const char* get_name() const { return "ErrorRateMeasure"; };
};

/** @brief class BALMeasure
* used to measure balanced error of 2-class classifier.
*
* This class is also capable of measuring
* any other rate using get_[measure name] methods
* of CContingencyTableEvaluation class.
*
* Note that evaluate() should be called first.
*/
class CBALMeasure: public CContingencyTableEvaluation
{
public:
/* constructor */
CBALMeasure() : CContingencyTableEvaluation(BAL) {};
/* virtual destructor */
virtual ~CBALMeasure() {};
/* name */
virtual inline const char* get_name() const { return "BALMeasure"; };
};

/** @brief class WRACCMeasure
* used to measure weighted relative accuracy of 2-class classifier.
*
* This class is also capable of measuring
* any other rate using get_[measure name] methods
* of CContingencyTableEvaluation class.
*
* Note that evaluate() should be called first.
*/
class CWRACCMeasure: public CContingencyTableEvaluation
{
public:
/* constructor */
CWRACCMeasure() : CContingencyTableEvaluation(WRACC) {};
/* virtual destructor */
virtual ~CWRACCMeasure() {};
/* name */
virtual inline const char* get_name() const { return "WRACCMeasure"; };
};

/** @brief class F1Measure
* used to measure F1 score of 2-class classifier.
*
* This class is also capable of measuring
* any other rate using get_[measure name] methods
* of CContingencyTableEvaluation class.
*
* Note that evaluate() should be called first.
*/
class CF1Measure: public CContingencyTableEvaluation
{
public:
/* constructor */
CF1Measure() : CContingencyTableEvaluation(F1) {};
/* virtual destructor */
virtual ~CF1Measure() {};
/* name */
virtual inline const char* get_name() const { return "F1Measure"; };
};

/** @brief class CrossCorrelationMeasure
* used to measure cross correlation coefficient of 2-class classifier.
*
* This class is also capable of measuring
* any other rate using get_[measure name] methods
* of CContingencyTableEvaluation class.
*
* Note that evaluate() should be called first.
*/
class CCrossCorrelationMeasure: public CContingencyTableEvaluation
{
public:
/* constructor */
CCrossCorrelationMeasure() : CContingencyTableEvaluation(CROSS_CORRELATION) {};
/* virtual destructor */
virtual ~CCrossCorrelationMeasure() {};
/* name */
virtual inline const char* get_name() const { return "CrossCorrelationMeasure"; };
};

/** @brief class RecallMeasure
* used to measure recall of 2-class classifier.
*
* This class is also capable of measuring
* any other rate using get_[measure name] methods
* of CContingencyTableEvaluation class.
*
* Note that evaluate() should be called first.
*/
class CRecallMeasure: public CContingencyTableEvaluation
{
public:
/* constructor */
CRecallMeasure() : CContingencyTableEvaluation(RECALL) {};
/* virtual destructor */
virtual ~CRecallMeasure() {};
/* name */
virtual inline const char* get_name() const { return "RecallMeasure"; };
};

/** @brief class PrecisionMeasure
* used to measure precision of 2-class classifier.
*
* This class is also capable of measuring
* any other rate using get_[measure name] methods
* of CContingencyTableEvaluation class.
*
* Note that evaluate() should be called first.
*/
class CPrecisionMeasure: public CContingencyTableEvaluation
{
public:
/* constructor */
CPrecisionMeasure() : CContingencyTableEvaluation(PRECISION) {};
/* virtual destructor */
virtual ~CPrecisionMeasure() {};
/* name */
virtual inline const char* get_name() const { return "PrecisionMeasure"; };
};

/** @brief class SpecificityMeasure
* used to measure specificity of 2-class classifier.
*
* This class is also capable of measuring
* any other rate using get_[measure name] methods
* of CContingencyTableEvaluation class.
*
* Note that evaluate() should be called first.
*/
class CSpecificityMeasure: public CContingencyTableEvaluation
{
public:
/* constructor */
CSpecificityMeasure() : CContingencyTableEvaluation(SPECIFICITY) {};
/* virtual destructor */
virtual ~CSpecificityMeasure() {};
/* name */
virtual inline const char* get_name() const { return "SpecificityMeasure"; };
};

}


#endif /* CONTINGENCYTABLEEVALUATION_H_ */
10 changes: 5 additions & 5 deletions src/shogun/evaluation/PRCEvaluation.cpp
View file
Expand Up @@ -23,7 +23,7 @@ float64_t CPRCEvaluation::evaluate(CLabels* predicted, CLabels* ground_truth)
{
ASSERT(predicted && ground_truth);
ASSERT(predicted->get_num_labels()==ground_truth->get_num_labels());
ASSERT(predicted->get_label_type()==LT_REAL);
ASSERT(predicted->get_label_type()==LT_BINARY);
ASSERT(ground_truth->get_label_type()==LT_BINARY);

// number of true positive examples
Expand All @@ -34,7 +34,7 @@ float64_t CPRCEvaluation::evaluate(CLabels* predicted, CLabels* ground_truth)
int32_t pos_count=0;

// initialize number of labels and labels
SGVector<float64_t> orig_labels = ((CRegressionLabels*) predicted)->get_labels();
SGVector<float64_t> orig_labels = predicted->get_confidences();
int32_t length = orig_labels.vlen;
float64_t* labels = CMath::clone_vector(orig_labels.vector, length);

Expand All @@ -55,7 +55,7 @@ float64_t CPRCEvaluation::evaluate(CLabels* predicted, CLabels* ground_truth)
// get total numbers of positive and negative labels
for (i=0; i<length; i++)
{
if (((CBinaryLabels*) ground_truth)->get_label(i) > 0)
if (ground_truth->get_confidence(i) > 0)
pos_count++;
}

Expand All @@ -66,15 +66,15 @@ float64_t CPRCEvaluation::evaluate(CLabels* predicted, CLabels* ground_truth)
for (i=0; i<length; i++)
{
// update number of true positive examples
if (((CBinaryLabels*) ground_truth)->get_label(idxs[i]) > 0)
if (ground_truth->get_confidence(idxs[i]) > 0)
tp += 1.0;

// precision (x)
m_PRC_graph[2*i] = tp/float64_t(i+1);
// recall (y)
m_PRC_graph[2*i+1] = tp/float64_t(pos_count);

m_thresholds[i]= ((CRegressionLabels*) predicted)->get_label(idxs[i]);
m_thresholds[i]= predicted->get_confidence(idxs[i]);
}

// calc auRPC using area under curve
Expand Down
1 change: 1 addition & 0 deletions src/shogun/evaluation/PRCEvaluation.h
View file
Expand Up @@ -66,6 +66,7 @@ class CPRCEvaluation: public CBinaryClassEvaluation
* @return thresholds
*/
SGVector<float64_t> get_thresholds();

protected:

/** 2-d array used to store PRC graph */
Expand Down
11 changes: 5 additions & 6 deletions src/shogun/evaluation/ROCEvaluation.cpp
View file
Expand Up @@ -9,8 +9,6 @@
*/

#include <shogun/evaluation/ROCEvaluation.h>
#include <shogun/labels/RegressionLabels.h>
#include <shogun/labels/BinaryLabels.h>
#include <shogun/mathematics/Math.h>

using namespace shogun;
Expand All @@ -23,7 +21,8 @@ float64_t CROCEvaluation::evaluate(CLabels* predicted, CLabels* ground_truth)
{
ASSERT(predicted && ground_truth);
ASSERT(predicted->get_num_labels()==ground_truth->get_num_labels());
ASSERT(predicted->get_label_type()==LT_REAL && ground_truth->get_label_type()==LT_BINARY);
ASSERT(predicted->get_label_type()==LT_BINARY);
ASSERT(ground_truth->get_label_type()==LT_BINARY);

// assume threshold as negative infinity
float64_t threshold = CMath::ALMOST_NEG_INFTY;
Expand Down Expand Up @@ -69,7 +68,7 @@ float64_t CROCEvaluation::evaluate(CLabels* predicted, CLabels* ground_truth)
// get total numbers of positive and negative labels
for(i=0; i<length; i++)
{
if (((CBinaryLabels*) ground_truth)->get_label(i) > 0)
if (ground_truth->get_confidence(i) >= 0)
pos_count++;
else
neg_count++;
Expand All @@ -84,7 +83,7 @@ float64_t CROCEvaluation::evaluate(CLabels* predicted, CLabels* ground_truth)
// create ROC curve and calculate auROC
for(i=0; i<length; i++)
{
label = ((CRegressionLabels*) predicted)->get_label(idxs[i]);
label = predicted->get_confidence(idxs[i]);

if (label != threshold)
{
Expand All @@ -96,7 +95,7 @@ float64_t CROCEvaluation::evaluate(CLabels* predicted, CLabels* ground_truth)

m_thresholds[i]=threshold;

if (((CBinaryLabels*) ground_truth)->get_label(idxs[i]) > 0)
if (ground_truth->get_confidence(idxs[i]) > 0)
tp+=1.0;
else
fp+=1.0;
Expand Down
2 changes: 1 addition & 1 deletion src/shogun/evaluation/ROCEvaluation.h
View file
Expand Up @@ -19,7 +19,7 @@ namespace shogun
class CLabels;

/** @brief Class ROCEvalution used to evaluate ROC
* (Receiver Operator Characteristic) and an area
* (Receiver Operating Characteristic) and an area
* under ROC curve (auROC).
*
* Implementation is based on the efficient ROC algorithm as described in
Expand Down
13 changes: 5 additions & 8 deletions src/shogun/labels/BinaryLabels.cpp
View file
Expand Up @@ -3,27 +3,24 @@

using namespace shogun;

CBinaryLabels::CBinaryLabels() : CDenseLabels(), m_threshold(0.0)
CBinaryLabels::CBinaryLabels() : CDenseLabels()
{
}

CBinaryLabels::CBinaryLabels(int32_t num_labels) : CDenseLabels(num_labels),
m_threshold(0.0)
CBinaryLabels::CBinaryLabels(int32_t num_labels) : CDenseLabels(num_labels)
{
}

CBinaryLabels::CBinaryLabels(SGVector<float64_t> src) : CDenseLabels(),
m_threshold(0.0)
CBinaryLabels::CBinaryLabels(SGVector<float64_t> src, float64_t threshold) : CDenseLabels()
{
SGVector<float64_t> labels(src.vlen);
for (int32_t i=0; i<labels.vlen; i++)
labels[i] = CMath::sign(src[i]+m_threshold);
labels[i] = src[i]+threshold>=0 ? +1.0 : -1.0;
set_labels(labels);
set_confidences(src);
}

CBinaryLabels::CBinaryLabels(CFile* loader) : CDenseLabels(loader),
m_threshold(0.0)
CBinaryLabels::CBinaryLabels(CFile* loader) : CDenseLabels(loader)
{
}

Expand Down

0 comments on commit 17f015e

Please sign in to comment.