Skip to content

Commit

Permalink
[DB MIGRATION REQ'D] Insights feed
Browse files Browse the repository at this point in the history 
* Create insight baseline model and DAO
* Calculate 7-day and 30-day retweet averages at the end of each Twitter crawl
* Generate post retweet and least likely follower at the end of each Twitter crawl
* Generate follower count history milestones to Sundays and day 1 of the month
* Generate flashback posts from this day 1 through 6 years ago
* Generate new list membership alerts and count chart
* Show Google Map when replies are geoencoded
* Generate most frequent reply keywords alerts
* Generate list membership insights
[ci skip]
  • Loading branch information
@ginatrapani
ginatrapani committed Jun 21, 2012
1 parent 1d00bbf commit 1abd362
Show file tree
Hide file tree
Showing 38 changed files with 1,443 additions and 48 deletions.
8 changes: 7 additions & 1 deletion tests/TestOfDAOFactory.php
View file
Expand Up @@ -376,6 +376,12 @@ public function testGetShortLinkDAO() {
$this->assertIsA($dao, 'ShortLinkMySQLDAO');
}

public function testInsightBaselineDAO() {
$dao = DAOFactory::getDAO('InsightBaselineDAO');
$this->assertNotNull($dao);
$this->assertIsA($dao, 'InsightBaselineMySQLDAO');
}

public function testInsightDAO() {
$dao = DAOFactory::getDAO('InsightDAO');
$this->assertNotNull($dao);
Expand All @@ -393,7 +399,7 @@ public function testGetInstallerDAONoConfigFile(){
$this->assertTrue(isset($dao));
$this->assertIsA($dao, 'InstallerMySQLDAO');
$result = $dao->getTables();
$this->assertEqual(sizeof($result), 30);
$this->assertEqual(sizeof($result), 31);
$this->assertEqual($result[0], $cfg_values["table_prefix"].'encoded_locations');
$this->restoreConfigFile();
}
Expand Down
73 changes: 69 additions & 4 deletions tests/TestOfFollowerCountMySQLDAO.php
View file
Expand Up @@ -117,6 +117,74 @@ public function testGetDayHistoryNoGapsMilestoneNotInSight() {
$this->assertNotNull($result['vis_data']);
}

public function testGetDayHistoryFromSpecificStartDate() {
$builders = array();
$format = 'n/j';
$date = date ( $format );

$todays_day_of_the_week = date('w');
$this->debug("It's currently the ".$todays_day_of_the_week." day of the week");
$follower_count = array('network_user_id'=>'930061', 'network'=>'twitter', 'date'=>'-40d', 'count'=>90);
$builders[] = FixtureBuilder::build('follower_count', $follower_count);

$follower_count = array('network_user_id'=>'930061', 'network'=>'twitter', 'date'=>'-41d', 'count'=>70);
$builders[] = FixtureBuilder::build('follower_count', $follower_count);

$follower_count = array('network_user_id'=>'930061', 'network'=>'twitter', 'date'=>'-42d', 'count'=>50);
$builders[] = FixtureBuilder::build('follower_count', $follower_count);

$follower_count = array('network_user_id'=>'930061', 'network'=>'twitter', 'date'=>'-43d', 'count'=>30);
$builders[] = FixtureBuilder::build('follower_count', $follower_count);

$follower_count = array('network_user_id'=>'930061', 'network'=>'twitter', 'date'=>'-44d', 'count'=>10);
$builders[] = FixtureBuilder::build('follower_count', $follower_count);

$dao = new FollowerCountMySQLDAO();
$date_ago = date ($format, strtotime('-40 day'.$date));
$result = $dao->getHistory('930061', 'twitter', 'DAY', 5, $date_ago);
$this->assertEqual(sizeof($result), 4, '4 sets of data returned--history, trend, and milestone, and vis_data');

$this->debug(Utils::varDumpToString($result));
//check history
$this->assertEqual(sizeof($result['history']), 5);

$format = 'm/d/Y';
if ($todays_day_of_the_week == 0 ) {
$date_ago = date ($format, strtotime('-40 day'.$date));
$this->assertEqual($result['history'][$date_ago], 90);

$date_ago = date ($format, strtotime('-41 day'.$date));
$this->assertEqual($result['history'][$date_ago], 70);

$date_ago = date ($format, strtotime('-42 day'.$date));
$this->assertEqual($result['history'][$date_ago], 50);

$date_ago = date ($format, strtotime('-43 day'.$date));
$this->assertEqual($result['history'][$date_ago], 30);
} else {
$date_ago = date ($format, strtotime('-41 day'.$date));
$this->assertEqual($result['history'][$date_ago], 70);

$date_ago = date ($format, strtotime('-42 day'.$date));
$this->assertEqual($result['history'][$date_ago], 50);

$date_ago = date ($format, strtotime('-43 day'.$date));
$this->assertEqual($result['history'][$date_ago], 30);
}

//check trend
$this->assertEqual($result['trend'], 16);

//check milestone
//latest follower count is 90, next milestone is 100 followers
//with a 20+/day trend, this should take 1 day
$this->debug(Utils::varDumpToString($result['milestone']));
$this->assertEqual($result['milestone']["next_milestone"], 100);
$this->assertEqual($result['milestone']["will_take"], 1);

$this->assertNotNull($result['vis_data']);
}

public function testGetDayHistoryNoGapsMilestoneInSight() {
$format = 'n/j';
$date = date ( $format );
Expand Down Expand Up @@ -249,11 +317,8 @@ public function testGetDayHistoryWeekNoGaps() {
//$this->assertEqual($result['trend'], 3);

//check milestone
//latest follower count is 140, next milestone is 1,000 followers
//with a 7+/day trend, this should take 123 days
//beyond our "don't feel bad about yourself" threshold of 10, so should be null
if ($todays_day_of_the_week != 0) {
$this->assertNull($result['milestone']);
$this->assertEqual($result['milestone']['will_take'], 20);
}
}

Expand Down
11 changes: 10 additions & 1 deletion tests/TestOfGroupMemberMySQLDAO.php
View file
Expand Up @@ -70,7 +70,7 @@ protected function buildData() {

// Jack's in two groups
$builders[] = FixtureBuilder::build('group_members', array('member_user_id'=>'1234567890',
'group_id'=>'18864710', 'is_active' => 1, 'network'=>'twitter', 'last_seen' => '-1h'));
'group_id'=>'18864710', 'is_active' => 1, 'network'=>'twitter', 'last_seen' => '-1h', 'first_seen' => '-1h'));

// one stale
$builders[] = FixtureBuilder::build('group_members', array('member_user_id'=>'1234567890',
Expand Down Expand Up @@ -147,4 +147,13 @@ public function testFindStalestMemberships() {
$stale_group = $this->DAO->findStalestMemberships($user_id = '1234567890', 'twitter');
$this->assertNull($stale_group);
}

public function testGetNewMembershipsByDate() {
$new_groups = $this->DAO->getNewMembershipsByDate('twitter', '1234567890');
$this->assertEqual(count($new_groups), 1);
$this->assertEqual($new_groups[0]->id, 1);
$this->assertEqual($new_groups[0]->group_id, '18864710');
$this->assertEqual($new_groups[0]->group_name, '@someguy/a-list');
$this->assertEqual($new_groups[0]->is_active, 1);
}
}
15 changes: 15 additions & 0 deletions tests/TestOfGroupMySQLDAO.php
View file
Expand Up @@ -113,4 +113,19 @@ public function testInsert() {
$this->assertTrue($this->DAO->updateOrInsertGroup($group));
$this->assertTrue($this->DAO->isGroupInStorage($group = '133333333', 'twitter'));
}

public function testSetMetadata() {
$val['id'] = 101;
$val['group_id'] = 1001;
$val['network'] = 'twitter';
$val['group_name'] = '@mirqamar/fivebyfive';
$val['is_active'] = 1;
$val['first_seen'] = '2012-06-13';
$val['last_seen'] = '2012-06-13';

$group = new Group($val);
$group->setMetadata();
$this->assertEqual($group->url, 'http://twitter.com/mirqamar/fivebyfive');
$this->assertEqual($group->keyword, 'fivebyfive');
}
}
104 changes: 104 additions & 0 deletions tests/TestOfInsightBaselineMySQLDAO.php
View file
@@ -0,0 +1,104 @@
<?php
/**
*
* ThinkUp/tests/TestOfInsightBaselineMySQLDAO.php
*
* Copyright (c) 2012 Gina Trapani
*
* LICENSE:
*
* This file is part of ThinkUp (http://thinkupapp.com).
*
* ThinkUp is free software: you can redistribute it and/or modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any
* later version.
*
* ThinkUp is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with ThinkUp. If not, see
* <http://www.gnu.org/licenses/>.
*
* @license http://www.gnu.org/licenses/gpl.html
* @copyright 2012 Gina Trapani
* @author Gina Trapani
*/
require_once dirname(__FILE__).'/init.tests.php';
require_once THINKUP_WEBAPP_PATH.'_lib/extlib/simpletest/autorun.php';
require_once THINKUP_WEBAPP_PATH.'config.inc.php';

class TestOfInsightBaselineMySQLDAO extends ThinkUpUnitTestCase {
public function setUp() {
parent::setUp();
$this->builders = self::buildData();
}

protected function buildData() {
$builders = array();
$builders[] = FixtureBuilder::build('insight_baselines', array('date'=>'2012-05-01',
'slug'=>'avg_replies_per_week', 'instance_id'=>1, 'value'=>51));
return $builders;
}

public function tearDown() {
$this->builders = null;
parent::tearDown();
}

public function testInsertInsightBaseline() {
$dao = new InsightBaselineMySQLDAO();
//date specified
$result = $dao->insertInsightBaseline('avg_replies_per_week', 1, 51, '2012-05-05');
$this->assertTrue($result);

$result = $dao->getInsightBaseline('avg_replies_per_week', 1, '2012-05-05');
$this->assertEqual($result->value, 51);

//inserting existing baseline should update
$result = $dao->insertInsightBaseline('avg_replies_per_week', 1, 50, '2012-05-05');
$this->assertTrue($result);

//assert update was successful
$result = $dao->getInsightBaseline('avg_replies_per_week', 1, '2012-05-05');
$this->assertEqual($result->value, 50);

//no date specified
$result = $dao->insertInsightBaseline('avg_replies_per_week', 1, 4551);
$this->assertTrue($result);
$result = $dao->getInsightBaseline('avg_replies_per_week', 1);
$this->assertEqual($result->value, 4551);
}

public function testGetInsightBaseline() {
$dao = new InsightBaselineMySQLDAO();
$result = $dao->getInsightBaseline('avg_replies_per_week', 1, '2012-05-01');
$this->assertIsA($result, 'InsightBaseline');
$this->assertEqual($result->slug, 'avg_replies_per_week');
$this->assertEqual($result->date, '2012-05-01');
$this->assertEqual($result->instance_id, 1);
$this->assertEqual($result->value, 51);

$result = $dao->getInsightBaseline('avg_replies_per_week', 1, '2012-05-02');
$this->assertNull($result);
}

public function testUpdateInsightBaseline() {
$dao = new InsightBaselineMySQLDAO();

//update existing baseline
$result = $dao->updateInsightBaseline('avg_replies_per_week', 1, 101, '2012-05-01');
$this->assertTrue($result);
//check that value was updated
$result = $dao->getInsightBaseline('avg_replies_per_week', 1, '2012-05-01');
$this->assertEqual($result->value, 101);

//update nonexistent baseline
$result = $dao->updateInsightBaseline('avg_replies_per_week', 1, 101, '2012-05-10');
$this->assertFalse($result);

//no date specified
$result = $dao->updateInsightBaseline('avg_replies_per_week', 1, 101);
$this->assertFalse($result);
}
}
1 change: 0 additions & 1 deletion tests/TestOfInsightMySQLDAO.php
View file
Expand Up @@ -88,7 +88,6 @@ public function tearDown() {
$this->builders = null;
parent::tearDown();
}

public function testGetInsight() {
$dao = new InsightMySQLDAO();
$result = $dao->getInsight('avg_replies_per_week', 1, '2012-05-01');
Expand Down
19 changes: 17 additions & 2 deletions tests/TestOfInstaller.php
View file
Expand Up @@ -140,6 +140,8 @@ public function testInstallerShowTables() {
$config->getValue('table_prefix')."instances, ".
$config->getValue('table_prefix')."instances_twitter, ".
$config->getValue('table_prefix')."invites," .
$config->getValue('table_prefix')."insight_baselines," .
$config->getValue('table_prefix')."insights," .
//$config->getValue('table_prefix')."links," .
$config->getValue('table_prefix')."links_short," .
$config->getValue('table_prefix')."mentions," .
Expand Down Expand Up @@ -217,6 +219,8 @@ public function testInstallerCheckTable() {
$config->getValue('table_prefix')."instances, ".
$config->getValue('table_prefix')."instances_twitter, ".
$config->getValue('table_prefix')."invites," .
$config->getValue('table_prefix')."insight_baselines," .
$config->getValue('table_prefix')."insights," .
$config->getValue('table_prefix')."links_short," .
$config->getValue('table_prefix')."mentions," .
$config->getValue('table_prefix')."mentions_posts, " .
Expand Down Expand Up @@ -285,6 +289,8 @@ public function testDoThinkUpTablesExist() {
$config->getValue('table_prefix')."instances, ".
$config->getValue('table_prefix')."instances_twitter, ".
$config->getValue('table_prefix')."invites," .
$config->getValue('table_prefix')."insight_baselines," .
$config->getValue('table_prefix')."insights," .
$config->getValue('table_prefix')."mentions," .
$config->getValue('table_prefix')."mentions_posts, " .
$config->getValue('table_prefix')."owner_instances, ".
Expand Down Expand Up @@ -329,6 +335,8 @@ public function testInstallerIsThinkUpInstalled() {
$config->getValue('table_prefix')."instances, ".
$config->getValue('table_prefix')."instances_twitter, ".
$config->getValue('table_prefix')."invites," .
$config->getValue('table_prefix')."insight_baselines," .
$config->getValue('table_prefix')."insights," .
$config->getValue('table_prefix')."mentions," .
$config->getValue('table_prefix')."mentions_posts, " .
$config->getValue('table_prefix')."owner_instances, ".
Expand Down Expand Up @@ -410,6 +418,8 @@ public function testInstallerPopulateTables() {
$config->getValue('table_prefix')."instances, ".
$config->getValue('table_prefix')."instances_twitter, ".
$config->getValue('table_prefix')."invites," .
$config->getValue('table_prefix')."insight_baselines," .
$config->getValue('table_prefix')."insights," .
$config->getValue('table_prefix')."links," .
$config->getValue('table_prefix')."links_short," .
$config->getValue('table_prefix')."mentions," .
Expand Down Expand Up @@ -453,6 +463,8 @@ public function testInstallerPopulateTables() {
$config->getValue('table_prefix')."instances, ".
$config->getValue('table_prefix')."instances_twitter, ".
$config->getValue('table_prefix')."invites," .
$config->getValue('table_prefix')."insight_baselines," .
$config->getValue('table_prefix')."insights," .
$config->getValue('table_prefix')."mentions," .
$config->getValue('table_prefix')."mentions_posts, " .
$config->getValue('table_prefix')."owner_instances, ".
Expand Down Expand Up @@ -495,6 +507,8 @@ public function testInstallerPopulateTables() {
$config->getValue('table_prefix')."instances, ".
$config->getValue('table_prefix')."instances_twitter, ".
$config->getValue('table_prefix')."invites," .
$config->getValue('table_prefix')."insight_baselines," .
$config->getValue('table_prefix')."insights," .
$config->getValue('table_prefix')."links," .
$config->getValue('table_prefix')."links_short," .
$config->getValue('table_prefix')."mentions," .
Expand Down Expand Up @@ -561,8 +575,9 @@ public function testGetTablesToInstall(){
$tables = $installer->getTablesToInstall();

$expected_tables = array('encoded_locations', 'favorites', 'follower_count', 'follows', 'group_member_count',
'group_members', 'groups', 'hashtags', 'hashtags_posts', 'insights', 'instances',
'instances_twitter', 'invites', 'links', 'links_short', 'mentions', 'mentions_posts', 'options',
'group_members', 'groups', 'hashtags', 'hashtags_posts',
'insight_baselines', 'insights', 'instances', 'instances_twitter', 'invites',
'links', 'links_short', 'mentions', 'mentions_posts', 'options',
'owner_instances', 'owners', 'places','places_posts',
'plugins', 'post_errors', 'posts', 'stream_data', 'stream_procs', 'user_errors', 'users');
$this->assertIdentical($tables, $expected_tables);
Expand Down
2 changes: 1 addition & 1 deletion tests/TestOfInstallerMySQLDAO.php
View file
Expand Up @@ -82,7 +82,7 @@ public function testGetTables() {
$config_array = $config->getValuesArray();
$dao = new InstallerMySQLDAO($config_array);
$result = $dao->getTables();
$this->assertEqual(sizeof($result), 29);
$this->assertEqual(sizeof($result), 30);
$this->assertEqual($result[0], $config_array["table_prefix"].'encoded_locations');
}
public function testCheckTable() {
Expand Down
38 changes: 38 additions & 0 deletions tests/TestOfPostMySQLDAO.php
View file
Expand Up @@ -2960,4 +2960,42 @@ public function testGetOnThisDayFlashbackPosts(){
// Check the link URL was set correctly
$this->assertEqual($res[0]->links[0]->url, 'http://bit.ly/blahb');
}

public function testGetAverageRetweetCount() {
$builders = array();
//Add straight text posts
$counter = 1;
while ($counter < 40) {
$pseudo_minute = str_pad($counter, 2, "0", STR_PAD_LEFT);
if ($counter % 3 == 0) {
$source = '<a href="http://twitter.com" rel="nofollow">Tweetie for Mac</a>';
} else if ($counter % 3 == 1) {
$source = '<a href="http://twitter.com/tweetbutton" rel="nofollow">Tweet Button</a>';
} else {
$source = 'web';
}
$builders[] = FixtureBuilder::build('posts', array('id'=>$counter+256, 'post_id'=>$counter+256,
'author_user_id'=>'13', 'author_username'=>'ev', 'author_fullname'=>'Ev Williams',
'author_avatar'=>'avatar.jpg', 'post_text'=>'This is post '.$counter,
'source'=>$source, 'pub_date'=>'-'.$counter.'d', 'in_reply_to_user_id'=>null,
'reply_count_cache'=>($counter==10)?0:rand(0, 4), 'is_protected'=>0,
'retweet_count_cache'=>floor($counter/2), 'network'=>'twitter',
'old_retweet_count_cache' => floor($counter/3), 'in_rt_of_user_id' => null,
'in_reply_to_post_id'=>null, 'in_retweet_of_post_id'=>null, 'is_geo_encoded'=>0));
$counter++;
}

$dao = new PostMySQLDAO();
//without date (today)
$average_retweet_count = $dao->getAverageRetweetCount('ev', 'twitter', 7);
$this->assertEqual($average_retweet_count, 3);

//yesterday
$average_retweet_count = $dao->getAverageRetweetCount('ev', 'twitter', 7, date("Y-m-d", strtotime("-1 day")));
$this->assertEqual($average_retweet_count, 4);

//40 days ago
$average_retweet_count = $dao->getAverageRetweetCount('ev', 'twitter', 7, date("Y-m-d", strtotime("-40 day")));
$this->assertEqual($average_retweet_count, 17);
}
}

0 comments on commit 1abd362

Please sign in to comment.