########################
#rdfstats.pl - get some basic statistics from a RDF file
######################3#
use RDF::Redland;
use URI;
use File::Util;
    
my $parser = new RDF::Redland::Parser (undef, "application/rdf+xml") || 
	die "Failed to find parser\n";
    
$query_count=0;
$document_count=0;
$triple_count=0;
@query_documents; #number of queries in a document
@query_triples; #number of queries in a document

#count of triples in each query

#hash of all URIs
my %hash_uris;
#hash of just domains
my %hash_domains;
#hash of all path except last bit
my %hash_paths;
#hash of all dataty-es
my %hash_datatypes;

#hash of predicate URIs
my %hash_predicates;
#hash of subject URIs
my %hash_subjects;
#hash of object URIs
my %hash_objects;

#counting RDF statements
$count_rdf_statements = 0;
#counting RDF(S) statements
$count_rdfs_statements = 0;
#counting OWL statements
$count_owl_statements = 0;
#counting_dc_statement
$count_dc_statements = 0;
#counting FOAF statements
$count_foaf_statements = 0;
#counting SKOS statements
$count_skos_statements = 0;

#log HTML files that aren't RDF for later GRDDLing

$uri_count=0;
$literal_count=0;
$blank_count=0;

$subject_blank_count=0;
$subject_uri_count=0;
$predicate_blank_count=0;
$predicate_uri_count=0;
$object_uri_count=0;
$object_blank_count=0;
$object_literal_count=0;

#go into each directory and read all the RDF files
my($f) = File::Util->new();
my(@dirs) = $f->list_dir('.','--dirs-only');
shift(@dirs); #to get rid of . directory
shift(@dirs); #to get rid of .. directory
$triples_per_query=0;
$documents_per_query=0;

foreach $dir (@dirs)
{
	chdir($dir);
	#system('rm *.2.*');
	$query_triples[$query_count]=$triples_per_query;
	$triples_per_query=0;
	$query_documents[$query_count]=$documents_per_query;
	$documents_per_query=0;
	$query_count++;
	open(LOGOUTPUT, ">>../log.txt") or die "Cannot open log.txt\n";
	print LOGOUTPUT "In $dir\n";
	print "$dir\n";
	close(LOGOUTPUT);
	@rdf_files = <*.rdf>;
	foreach $rdf_file (@rdf_files)
	{
	$documents_per_query++;
	#delete any escape characters from filename
	if ($rdf_file =~ m/%/)
	{
		$old_file_name = $rdf_file;
		$rdf_file =~ s/(%\w\w)/__/gi;
		rename($old_file_name, $rdf_file);	
	}
	open(LOGOUTPUT, ">>../log.txt") or die "Cannot open log.txt\n";
	print LOGOUTPUT "$rdf_file\n";
	close(LOGOUTPUT);
	$document_count++;
	print "$rdf_file\n";
	my $uri    = new RDF::Redland::URI ("file:$rdf_file");
	eval {
	my $stream = $parser->parse_as_stream ($uri, $uri);
	while (!$stream->end) {
		my $statement=$stream->current;
		print $statement->as_string,"\n"; 
	        $triple_count++;
		$triples_per_query++;
		if ($statement->subject->is_resource){$subject_uri_count++;
		my $uri_string=$statement->subject->as_string;
		$uri_string =~ s/\[//;
		$uri_string =~ s/\]//;
	        if (defined $hash_subjects{$uri_string}) {
 		      $hash_subjects{$uri_string} = $hash_subjects{$uri_string}+1;}
        	else {$hash_subjects{$uri_string} = 1;}
		&checkURI($uri_string);
		}
   		elsif($statement->subject->is_blank){$subject_blank_count++;}
   		if($statement->predicate->is_resource){$predicate_uri_count++;
		my $uri_string=$statement->predicate->as_string;
		$uri_string =~ s/\[//;
		$uri_string =~ s/\]//;
	        if (defined $hash_predicates{$uri_string}) {
 		      $hash_predicates{$uri_string} = $hash_predicates{$uri_string}+1;}
        	else {$hash_predicates{$uri_string} = 1;}
		&checkURI($uri_string);
                }
		elsif($statement->predicate->is_blank){$predicate_blank_count++;}
   		if($statement->object->is_resource){$object_uri_count++;
		my $uri_string=$statement->object->as_string;
		$uri_string =~ s/\[//;
		$uri_string =~ s/\]//;
	        if (defined $hash_objects{$uri_string}) {
 		      $hash_objects{$uri_string} = $hash_objects{$uri_string}+1;}
        	else {$hash_objects{$uri_string} = 1;}
		&checkURI($uri_string);
                }
   		elsif($statement->object->is_blank){$object_blank_count++;}	
		elsif($statement->object->is_literal)
			{
                	$object_literal_count++;
			if($statement->object->literal_value_is_wf_xml>0){
				$object_xml_count++;
   			#print "LITERAL:",$statement->object->literal_value_language,"\n";
				}
			if (defined $statement->object->literal_datatype)
		        {
			my $uri_string=$statement->object->literal_datatype->as_string;
			$uri_string =~ s/\[//;
			$uri_string =~ s/\]//;
			if (defined $hash_datatypes{$uri_string}) 
 		      		{$hash_datatypes{$uri_string} = $hash_datatypes{$uri_string}+1;}
        		else {$hash_datatypes{$uri_string} = 1;}
			}
		}
		$stream->next;
		}
	$stream=undef;
	};
	if ($@) 
	{
	open(LOGOUTPUT, ">>../log.txt") or die "Cannot open log.txt\n";
	print LOGOUTPUT "$rdf_file: We had a PROBLEM! $@\n"; 
	print "$rdf_file: We had a PROBLEM! $@\n"; 
	close(LOGOUTPUT);
	$@='';
	}
	}
	chdir('..');
	$query_documents[$query_count]=$documents_per_query;
	$query_triples[$query_count]=$triples_per_query;
}


#now at output stage
$total_uri_count=$object_uri_count+$subject_uri_count+$predicate_uri_count;
$total_blank_count=$object_blank_count+$subject_blank_count+$predicate_blank_count;
$total_literal_count=$object_literal_count;
$total_xml_count=$object_xml_count;
$total_string_count=$object_literal_count-$object_xml_count;



open(DOCOUTPUT, ">query_uricount.txt") or die "Cannot open query_uricount.txt\n";
foreach $query_document (@query_documents)
{print DOCOUTPUT "$query_document \n";}
close(DOCOUTPUT);

open(TRIOUTPUT, ">query_triplecount.txt") or die "Cannot open query_triplecount.txt\n";
foreach $query_triple (@query_triples)
{print TRIOUTPUT "$query_triple \n";}
close(TRIOUTPUT);

open(MAINOUTPUT, ">total_stats.txt") or die "Cannot open total_stats.txt\n";
print MAINOUTPUT "Total Queries:  $query_count \n";
print MAINOUTPUT "Total Documents: $document_count \n";
print MAINOUTPUT "Total Triples: $triple_count \n\n";
print MAINOUTPUT "Triple Count: $triple_count\n";    
print MAINOUTPUT "Total URIs: $total_uri_count\n";    
print MAINOUTPUT "Total Blank Nodes: $total_blank_count\n";    
print MAINOUTPUT "Total Literals: $total_literal_count\n";    
print MAINOUTPUT "Total XML Literals: $total_xml_count\n";    
print MAINOUTPUT "Total String Literals: \n";

print MAINOUTPUT "Subject URIs: $subject_uri_count\n";    
print MAINOUTPUT "Subject Blank Nodes: $subject_blank_count\n";    

print MAINOUTPUT "Predicate URIs: $predicate_uri_count\n";    
print MAINOUTPUT "Predicate Blank Nodes: $predicate_blank_count\n";    

print MAINOUTPUT "Object URIs: $object_uri_count\n";    
print MAINOUTPUT "Object Blank Nodes: $object_blank_count\n";    
print MAINOUTPUT "Object Literals: $object_literal_count\n";    

print MAINOUTPUT "RDF URIs: $count_rdf_statements\n";
print MAINOUTPUT "RDFS URIs: $count_rdfs_statements\n";
print MAINOUTPUT "OWL URIs: $count_owl_statements\n";
print MAINOUTPUT "SKOS URIs: $count_skos_statements\n";
print MAINOUTPUT "DC URIs: $count_dc_statements\n";
print MAINOUTPUT "FOAF URIs: $count_foaf_statements\n";

close(MAINOUTPUT);

open(URIOUTPUT, ">uri_list.txt") or die "Cannot open uri_list.txt\n";
print "All URIs\n";
foreach $value (reverse sort {$hash_uris{$a} <=> $hash_uris{$b} } keys %hash_uris)  {
    print URIOUTPUT "$hash_uris{$value}\t$value\n";}
close(URIOUTPUT);

open(DOMAINOUTPUT, ">domain_list.txt") or die "Cannot open domain_list.txt\n";
print "All Domains\n";
foreach $value (reverse sort {$hash_domains{$a} <=> $hash_domains{$b} } keys %hash_domains)  {
    print DOMAINOUTPUT "$hash_domains{$value}\t$value\n";}
close(DOMAINOUTPUT);

open(PATHOUTPUT, ">voc_list.txt") or die "Cannot open voc_list.txt\n";
foreach $value (reverse sort {$hash_paths{$a} <=> $hash_paths{$b} } keys %hash_paths)  {
    print PATHOUTPUT "$hash_paths{$value}\t$value\n";}

close(PATHOUTPUT);

open(SUBJECTOUTPUT, ">subject_uris.txt") or die "Cannot open subject_uris.txt\n";
print "Subject URIs\n";
foreach $value (reverse sort {$hash_subjects{$a} <=> $hash_subjects{$b} } keys %hash_subjects)  {
    print SUBJECTOUTPUT "$hash_subjects{$value}\t$value\n";}
close(SUBJECTOUTPUT);

open(PREDICATEOUTPUT, ">predicate_uris.txt") or die "Cannot open predicate_uris.txt\n";
print "Predicate URIs\n";
foreach $value (reverse sort {$hash_predicates{$a} <=> $hash_predicates{$b} }  keys %hash_predicates)  {
    print PREDICATEOUTPUT "$hash_predicates{$value}\t$value\n";
}
close(PREDICATEOUTPUT);

open(OBJECTOUTPUT, ">object_uris.txt") or die "Cannot open object_uris.txt\n";
print "Object URIs\n";
foreach $value (reverse sort {$hash_objects{$a} <=> $hash_objects{$b} } keys %hash_objects)  {
    print OBJECTOUTPUT "$hash_objects{$value}\t$value\n";
}
close(OBJECTOUTPUT);

open(DATATYPESOUTPUT, ">datatypes_uris.txt") or die "Cannot open datatypes_uris.txt\n";
print "Datatypes URIs\n";
foreach $value (reverse sort {$hash_datatypes{$a} <=> $hash_datatypes{$b} } keys %hash_datatypes)  {
    print DATATYPESOUTPUT "$hash_datatypes{$value}\t$value\n";
}
close(OBJECTOUTPUT);

sub checkURI
{
	my $uri_string=$_[0];
	$uri1 = URI->new("$uri_string");
	$domain=$uri1->host;
    	$path=$uri1->path;
    	$path =~ m/(\/.+\/)*(.+$)/;
	$voc_path=$1;
    	$voc_uri = 'http://'."$domain"."$voc_path";
        if (defined $hash_uris{ $uri_string}) {
 	      $hash_uris{$uri_string} = $hash_uris{$uri_string}+1;}
        else {$hash_uris{$uri_string} = 1;}
        if (defined $hash_domains{$domain}) {
 	      $hash_domains{$domain} = $hash_domains{$domain}+1;}
        else {$hash_domains{$domain} = 1;}
        if (defined $hash_paths{$voc_uri}) {
 	      $hash_paths{$voc_uri} = $hash_paths{$voc_uri}+1;}
        else {$hash_paths{$voc_uri} = 1;}
	if ($uri_string =~ m/rdf-syntax-ns/)
	{$count_rdf_statements++;}
	elsif ($uri_string =~ m/2000\/01\/rdf-schema/)
	{$count_rdfs_statements++;}
	elsif ($uri_string =~ m/2002\/07\/owl/)
	{$count_owl_statements++;}
	elsif ($uri_string =~ m/2004\/02\/skos/)
	{$count_skos_statements++;}
	elsif ($uri_string =~ m/xmlns\.com\/foaf/)
	{$count_foaf_statements++;}
	elsif ($uri_string =~ m/purl\.org\/dc/)
	{$count_dc_statements++;}
}
