json-counts

#!/usr/bin/env perl

use strict;
use warnings;
use Data::Dumper;
use File::Slurp qw(read_file);
use JSON::MaybeXS qw(decode_json);

# still used for validation
my $location_filename = 'location_history/your_location_history.json';

my ($data_dir) = @ARGV;
validate_dir($data_dir);

our @generic_index = (
	{
		key => 'history',
		filename => 'ads/ads_interests.json',
		typename => 'ad interests'
	},
	{
		key => 'history',
		filename => 'ads/advertisers_who_uploaded_a_contact_list_with_your_information.json',
		typename => 'advertiser uploaded contact'
	},
	{
		key => 'history',
		filename => 'ads/advertisers_you\'ve_interacted_with.json',
		typename => 'advertiser touched'
	},
	{
		key => 'installed_apps',
		filename => 'apps_and_websites/apps_and_websites.json',
		typename => 'installed_apps'
	},
	{
		key => 'app_posts',
		filename => 'apps_and_websites/posts_from_apps_and_websites.json',
		typename => 'app_post'
	},
	{
		key => 'comments',
		filename => 'comments/comments.json',
		typename => 'comment'
	},
	{
		key => 'events_invited',
		filename => 'events/event_invitations.json',
		typename => 'events_invited'
	},
	{
		key => 'pages_followed',
		filename => 'following_and_followers/followed_pages.json',
		typename => 'followed page'
	},
	{
		key => 'following',
		filename => 'following_and_followers/following.json',
		typename => 'following'
	},
	{
		key => 'friends',
		filename => 'friends/friends.json',
		typename => 'friends'
	},
	{
		key => 'received_requests',
		filename => 'friends/received_friend_requests.json',
		typename => 'received friends requests'
	},
	{
		key => 'rejected_requests',
		filename => 'friends/rejected_friend_requests.json',
		typename => 'rejected friends requests'
	},
	{
		key => 'sent_requests',
		filename => 'friends/sent_friend_requests.json',
		typename => 'sent friend requests'
	},
	{
		key => 'groups_joined',
		filename => 'groups/your_group_membership_activity.json',
		typename => 'groups joined'
	},
	{
		key => 'group_posts',
		filename => 'groups/your_posts_and_comments_in_groups.json',
		typename => 'group_posts'
	},
	{
		key => 'other_likes',
		filename => 'likes_and_reactions/likes_on_external_sites.json',
		typename => 'other like'
	},
	{
		key => 'page_likes',
		filename => 'likes_and_reactions/pages.json',
		typename => 'page like'
	},
	{
		key => 'reactions',
		filename => 'likes_and_reactions/posts_and_comments.json',
		typename => 'reaction'
	},
	{
		key => 'location_history',
		filename => 'location_history/your_location_history.json',
		typename => 'location'
	},
	{
		key => 'pokes',
		filename => 'other_activity/pokes.json',
		typename => 'pokes'
	},
	{
		key => 'pages',
		filename => 'pages/your_pages.json',
		typename => 'pages'
	},
	{
		key => 'wall_posts_sent_to_you',
		filename => 'posts/other_people\'s_posts_to_your_timeline.json',
		typename => 'wall_posts_sent_to_you'
	},
	{
		# TODO: status updaetes per year/month
		key => 'status_updates',
		filename => 'posts/your_posts.json',
		typename => 'status update'
	},
	{
		key => 'profile_updates',
		filename => 'profile_information/profile_update_history.json',
		typename => 'profile_updates'
	},
	{
		key => 'searches',
		filename => 'search_history/your_search_history.json',
		typename => 'searches'
	},
	{
		key => 'account_activity',
		filename => 'security_and_login_information/account_activity.json',
		typename => 'account_activity'
	},
	{
		key => 'admin_records',
		filename => 'security_and_login_information/administrative_records.json',
		typename => 'admin_records'
	},
	{
		key => 'recognized_devices',
		filename => 'security_and_login_information/authorized_logins.json',
		typename => 'recognized_devices'
	},
	{
		key => 'login_protection_data',
		filename => 'security_and_login_information/login_protection_data.json',
		typename => 'login_protection_data'
	},
	{
		key => 'account_accesses',
		filename => 'security_and_login_information/logins_and_logouts.json',
		typename => 'account_accesses'
	},
	{
		key => 'used_ip_address',
		filename => 'security_and_login_information/used_ip_addresses.json',
		typename => 'used_ip_address'
	},
	{
		key => 'active_sessions',
		filename => 'security_and_login_information/where_you\'re_logged_in.json',
		typename => "active session"
	},
	{
		key => 'your_places',
		filename => 'your_places/places_you\'ve_created.json',
		typename => "places you've created"
	},
	{
		key => '', # just a hash
		skip => 1,
		filename => 'about_you/face_recognition.json',
		typename => ''
	},
	{
		key => '', # tiny
		skip => 1,
		filename => 'about_you/friend_peer_group.json',
		typename => 'friend_peer_group'
	},
	{
		key => '', # address_book -> address_book
		skip => 1,
		filename => 'about_you/your_address_books.json',
		typename => 'address_book'
	},
	{
		key => '', # 3 subkeys
		skip => 1,
		filename => 'events/your_event_responses.json',
		typename => 'event_responses'
	},
	{
		key => '', # payments -> payments
		skip => 1,
		filename => 'payment_history/payment_history.json',
		typename => 'payments'
	},
	{
		key => '',
		skip => 1,
		filename => 'photos_and_videos/album/0.json',
		typename => ''
	},
	{
		key => '', # not countable
		skip => 1,
		filename => 'profile_information/profile_information.json',
		typename => 'profile'
	},
);

count_marketplace();
count_saved_items();

do_generics();
do_messages();

#
# count functions
#

sub count_saved_items {
	my $no_saved_items_filename = 'saved_items/no-data.txt';
	if (-f $no_saved_items_filename) {
		print "- 0 saved_items entries\n";
	} else {
		warn "unimplemented saved_items count"; # TODO: implement
	}
}

sub count_marketplace {
	my $no_marketplace_filename = 'marketplace/no-data.txt';
	if (-f $no_marketplace_filename) {
		print "- 0 marketplace entries\n";
	} else {
		warn "unimplemented marketplace count"; # TODO: implement
	}
}

#
# messages
#

sub do_messages {
	my $conversations = `find messages -name message.json`;
	my @conversations = split(/\n/,$conversations);
	my $conversation_count = scalar(@conversations);
	print "- $conversation_count conversations found in Facebook Messages\n";

	my $total_messages = 0;
	foreach my $convo_filename (@conversations) {
		my $conversation = read_json($convo_filename);
		my $title = $conversation->{'title'}; # usually the other person

		# count messages
		my @things = @{$conversation->{'messages'}};
		my $count = scalar @things;
		$total_messages += $count;

		# parse out bucket from filename
		my(undef, $bucket) = split(m{[/]}, $convo_filename);

		print "+ $count messages with $title ($bucket)\n";
	}
	print "- $total_messages messages in those $conversation_count conversations\n";
}

#
# generics
#

sub do_generics {
	foreach my $generic (@generic_index) {
		my $key = $generic->{key};
		my $filename = $generic->{filename};
		my $typename = $generic->{typename};

		my $skip = $generic->{skip} || 0;
		if ($skip) {
			print "... skipping $filename which isn't generic enough to process yet.\n";
			next; # skip!
		}

		if (length $key) {
			count_generic($filename,$typename,$key);
		} else {
			top_keys_file($filename);
		}
	}
}

sub count_generic {
	my ($filename,$type,$key) = @_;

	unless (-f $filename) {
		warn "no such file as $filename";
		return;
	}
	my $hash = read_json($filename);

	my @things = @{$hash->{$key}};

	my $count = scalar @things;
	print "- $count $type entries\n";
}

#
# utility functions
#

sub top_keys {
	my ($hash) = @_;

	my @keys = keys %$hash;

	print "  % " . join("\n  % ", @keys) . "\n";
}

sub top_keys_file {
	my ($filename) = @_;

	my $hash = read_json($filename);
	print " + $filename :\n";
	top_keys($hash);
}

sub read_json {
	my ($json_filename) = @_;
	my $file_contents = read_file($json_filename);

	my $json = JSON::MaybeXS->new->allow_nonref;
	my $js_ref = $json->decode($file_contents);
	die "no ref back from json->decode()" unless $js_ref;

	return $js_ref;
}

#
# sanity check
#

sub validate_dir {
	my ($data_dir) = @_;

	# is it a directory???
	unless (-d $data_dir) {
		die "not a directory: $data_dir";
	}

	chdir($data_dir) or die "couldn't chdir($data_dir): $!";

	# does it contain the right subdirectories???
	my @top_dirs = qw( about_you ads apps_and_websites calls_and_messages
			comments events following_and_followers friends
			groups likes_and_reactions location_history
			marketplace messages other_activity pages
			payment_history photos_and_videos posts
			profile_information saved_items search_history
			security_and_login_information your_places);
	foreach my $dir (@top_dirs) {
		unless (-d $dir) {
			die "no $dir directory in $data_dir";
		}
	}

	# how about a file?
	unless (-f $location_filename) {
		die "no  $location_filename";
	}

	print "$data_dir looks like a Facebook data dump\n";
}