Skip to content

Commit

Permalink
Insight generation process tweaks
Browse files Browse the repository at this point in the history 
Get follower count history before a specified date to backfill follower count insights
Show follower count history rate on insight feed item
Add paging to insights feed, and "No insights to display" message if there are no insights
Generate insights and baselines for a configurable number of backfilled days
Default to 3 days back on every Twitter crawl insights generation
[ci skip]
  • Loading branch information
@ginatrapani
ginatrapani committed May 21, 2012
1 parent 1eba0aa commit 2f6c063
Show file tree
Hide file tree
Showing 7 changed files with 214 additions and 79 deletions.
73 changes: 69 additions & 4 deletions tests/TestOfFollowerCountMySQLDAO.php
View file
Expand Up @@ -117,6 +117,74 @@ public function testGetDayHistoryNoGapsMilestoneNotInSight() {
$this->assertNotNull($result['vis_data']);
}

public function testGetDayHistoryFromSpecificStartDate() {
$builders = array();
$format = 'n/j';
$date = date ( $format );

$todays_day_of_the_week = date('w');
$this->debug("It's currently the ".$todays_day_of_the_week." day of the week");
$follower_count = array('network_user_id'=>'930061', 'network'=>'twitter', 'date'=>'-40d', 'count'=>90);
$builders[] = FixtureBuilder::build('follower_count', $follower_count);

$follower_count = array('network_user_id'=>'930061', 'network'=>'twitter', 'date'=>'-41d', 'count'=>70);
$builders[] = FixtureBuilder::build('follower_count', $follower_count);

$follower_count = array('network_user_id'=>'930061', 'network'=>'twitter', 'date'=>'-42d', 'count'=>50);
$builders[] = FixtureBuilder::build('follower_count', $follower_count);

$follower_count = array('network_user_id'=>'930061', 'network'=>'twitter', 'date'=>'-43d', 'count'=>30);
$builders[] = FixtureBuilder::build('follower_count', $follower_count);

$follower_count = array('network_user_id'=>'930061', 'network'=>'twitter', 'date'=>'-44d', 'count'=>10);
$builders[] = FixtureBuilder::build('follower_count', $follower_count);

$dao = new FollowerCountMySQLDAO();
$date_ago = date ($format, strtotime('-40 day'.$date));
$result = $dao->getHistory('930061', 'twitter', 'DAY', 5, $date_ago);
$this->assertEqual(sizeof($result), 4, '4 sets of data returned--history, trend, and milestone, and vis_data');

$this->debug(Utils::varDumpToString($result));
//check history
$this->assertEqual(sizeof($result['history']), 5);

$format = 'm/d/Y';
if ($todays_day_of_the_week == 0 ) {
$date_ago = date ($format, strtotime('-40 day'.$date));
$this->assertEqual($result['history'][$date_ago], 90);

$date_ago = date ($format, strtotime('-41 day'.$date));
$this->assertEqual($result['history'][$date_ago], 70);

$date_ago = date ($format, strtotime('-42 day'.$date));
$this->assertEqual($result['history'][$date_ago], 50);

$date_ago = date ($format, strtotime('-43 day'.$date));
$this->assertEqual($result['history'][$date_ago], 30);
} else {
$date_ago = date ($format, strtotime('-41 day'.$date));
$this->assertEqual($result['history'][$date_ago], 70);

$date_ago = date ($format, strtotime('-42 day'.$date));
$this->assertEqual($result['history'][$date_ago], 50);

$date_ago = date ($format, strtotime('-43 day'.$date));
$this->assertEqual($result['history'][$date_ago], 30);
}

//check trend
$this->assertEqual($result['trend'], 16);

//check milestone
//latest follower count is 90, next milestone is 100 followers
//with a 20+/day trend, this should take 1 day
$this->debug(Utils::varDumpToString($result['milestone']));
$this->assertEqual($result['milestone']["next_milestone"], 100);
$this->assertEqual($result['milestone']["will_take"], 1);

$this->assertNotNull($result['vis_data']);
}

public function testGetDayHistoryNoGapsMilestoneInSight() {
$format = 'n/j';
$date = date ( $format );
Expand Down Expand Up @@ -249,11 +317,8 @@ public function testGetDayHistoryWeekNoGaps() {
//$this->assertEqual($result['trend'], 3);

//check milestone
//latest follower count is 140, next milestone is 1,000 followers
//with a 7+/day trend, this should take 123 days
//beyond our "don't feel bad about yourself" threshold of 10, so should be null
if ($todays_day_of_the_week != 0) {
$this->assertNull($result['milestone']);
$this->assertEqual($result['milestone']['will_take'], 20);
}
}

Expand Down
78 changes: 57 additions & 21 deletions webapp/_lib/model/class.FollowerCountMySQLDAO.php
View file
Expand Up @@ -43,7 +43,11 @@ public function insert($network_user_id, $network, $count){
return $this->getInsertCount($ps);
}

public function getHistory($network_user_id, $network, $units, $limit=10) {
public function getHistory($network_user_id, $network, $units, $limit=10, $before_date=null) {
if ($before_date == date('Y-m-d')) {
$before_date = null;
}

if ($units != "DAY" && $units != 'WEEK' && $units != 'MONTH') {
$units = 'DAY';
}
Expand All @@ -57,45 +61,50 @@ public function getHistory($network_user_id, $network, $units, $limit=10) {
$group_by = 'YEAR(fc.date), MONTH(fc.date)';
$date_format = "DATE_FORMAT(date,'%m/01/%Y')";
}
$vars = array(
':network_user_id'=>(string) $network_user_id,
':network'=>$network,
':limit'=>(int)$limit
);
$q = "SELECT network_user_id, network, count, date, full_date FROM ";
$q .= "(SELECT network_user_id, network, count, ".$date_format." as date, date as full_date ";
$q .= "FROM #prefix#follower_count AS fc ";
$q .= "WHERE fc.network_user_id = :network_user_id AND fc.network=:network ";
if ($before_date != null) {
$q .= "AND date <= :before_date ";
$vars[':before_date'] = $before_date;
}
$q .= "GROUP BY ".$group_by." ORDER BY full_date DESC LIMIT :limit ) as history_counts ";
$q .= "ORDER BY history_counts.full_date ASC";
$vars = array(
':network_user_id'=>(string) $network_user_id,
':network'=>$network,
':limit'=>(int)$limit
);
if ($this->profiler_enabled) Profiler::setDAOMethod(__METHOD__);

$ps = $this->execute($q, $vars);
$history_rows = $this->getDataRowsAsArrays($ps);

$resultset = array();
switch ($network) {
case 'facebook':
$follower_description = 'Friends';
break;
case 'facebook page':
$follower_description = 'Fans';
break;
case 'twitter':
default:
$follower_description = 'Followers';
break;
case 'facebook':
$follower_description = 'Friends';
break;
case 'facebook page':
$follower_description = 'Fans';
break;
case 'twitter':
default:
$follower_description = 'Followers';
break;
}
foreach ($history_rows as $row) {
$timestamp = strtotime($row['full_date']);
$resultset[] = array('c' => array(
array('v' => sprintf('new Date(%d,%d,%d)', date('Y', $timestamp), date('n', $timestamp) - 1,
date('j', $timestamp)), 'f' => $row['date']),
array('v' => intval($row['count']))
array('v' => sprintf('new Date(%d,%d,%d)', date('Y', $timestamp), date('n', $timestamp) - 1,
date('j', $timestamp)), 'f' => $row['date']),
array('v' => intval($row['count']))
));
}
$metadata = array(
array('type' => 'date', 'label' => 'Date'),
array('type' => 'number', 'label' => $follower_description),
array('type' => 'date', 'label' => 'Date'),
array('type' => 'number', 'label' => $follower_description),
);
$vis_data = json_encode(array('rows' => $resultset, 'cols' => $metadata));
// Google Chart docs say that a string of the form "Date(Y,m,d)" should
Expand Down Expand Up @@ -156,6 +165,33 @@ public function getHistory($network_user_id, $network, $units, $limit=10) {

$milestone = Utils::predictNextMilestoneDate(intval($history_rows[sizeof($history_rows)-1]['count']),
$trend);

//If $before_date set, add difference between then and now to how long it will take
if (isset($before_date) ) {
if ($units=='DAY') {
$current_day_of_year = date('z');
$before_date_day_of_year = date('z', strtotime($before_date));
$difference = $current_day_of_year - $before_date_day_of_year;
if ($milestone['will_take'] > $difference) {
$milestone['will_take'] = $milestone['will_take'] + $difference;
}
} elseif ($units=='WEEK') {
$current_week_of_year = date('W');
$before_date_week_of_year = date('W', strtotime($before_date));
$difference = $current_week_of_year - $before_date_week_of_year;
if ($milestone['will_take'] > $difference) {
$milestone['will_take'] = $milestone['will_take'] + $difference;
}
} elseif ($units=='MONTH') {
$current_day_of_year = date('n');
$before_date_day_of_year = date('n', strtotime($before_date));
$difference = $current_month_of_year - $before_date_month_of_year;
if ($milestone['will_take'] > $difference) {
$milestone['will_take'] = $milestone['will_take'] + $difference;
}
}
}

if (isset($milestone)) {
$milestone['units_of_time'] = $units;
}
Expand Down
3 changes: 2 additions & 1 deletion webapp/_lib/model/interface.FollowerCountDAO.php
View file
Expand Up @@ -45,7 +45,8 @@ public function insert($network_user_id, $network, $count);
* @param str $network
* @param str $group_by 'DAY', 'WEEK', 'MONTH'
* @param int $limit Defaults to 10
* @param str $start_date Defaults to null (today)
* @return array $history, $percentages
*/
public function getHistory($network_user_id, $network, $group_by, $limit=10);
public function getHistory($network_user_id, $network, $group_by, $limit=10, $before_date=null);
}
5 changes: 5 additions & 0 deletions webapp/_lib/view/_insights.follower_count_history.tpl
View file
@@ -1,4 +1,5 @@
<br><br>

<div id="follower_count_history_{$i->id}"></div>

<script type="text/javascript">
Expand Down Expand Up @@ -46,4 +47,8 @@ function drawChart{/literal}{$i->id}() {literal}{
}
{/literal}
</script>
{if $i->related_data.trend != 0}
Current growth rate: {if $i->related_data.trend > 0}<span style="color:green">+{else}<span style="color:red">{/if}{$i->related_data.trend|number_format}</span>/week
{/if}
<br><br>
17 changes: 16 additions & 1 deletion webapp/_lib/view/insights.tpl
View file
@@ -1,5 +1,10 @@
<h1>Insights (alpha)</h1>

{if sizeof($insights) eq 0}
<div class="alert urgent">
<p>No insights are available! Get active on your network and check back later.</p>
</div>
{/if}
{assign var='cur_date' value=''}
{foreach from=$insights key=tid item=i name=foo}
{if $cur_date neq $i->date}
Expand All @@ -24,4 +29,14 @@
</span>
</p>
</div>
{/foreach}
{/foreach}

<div class="view-all" id="older-posts-div">
{if $next_page}
<a href="{$site_root_path}?{if $smarty.get.v}v={$smarty.get.v}&{/if}{if $smarty.get.u}u={$smarty.get.u}&{/if}{if $smarty.get.n}n={$smarty.get.n|urlencode}&{/if}page={$next_page}" id="next_page">&#60; Older</a>
{/if}
{if $last_page}
| <a href="{$site_root_path}?{if $smarty.get.v}v={$smarty.get.v}&{/if}{if $smarty.get.u}u={$smarty.get.u}&{/if}{if $smarty.get.n}n={$smarty.get.n|urlencode}&{/if}page={$last_page}" id="last_page">Newer &#62;</a>
{/if}
</div>

0 comments on commit 2f6c063

Please sign in to comment.