#!perl -w #Copyright (c) 20005, The University of Manchester, UK #All rights reserved. # #Redistribution and use in source and binary forms, with or without modification, #are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of the The University of Manchester nor the names of its contributors # may be used to endorse or promote products derived from this software without # specific prior written permission. # #THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" #AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, #THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE #ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS #BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES #(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; #LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON #ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT #(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, #EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Usage Record Spooler (urspool.pl) # John Ainsworth January 2005 # The University of Manchester # Finds, manipulates, inserts and deletes Usage Records # Site specific code, but can easily be adapted use lib ("SOAP-Lite-0.60/lib", "/usr/local/apps/lib/perl5/site_perl/5.6.1"); use SOAP::Lite on_fault => sub{}; use LWP::Protocol::https; use SOAP::Lite;# +trace => 'debug'; use Crypt::SSLeay; use IO::Socket::SSL; #site specific $ENV{HTTPS_VERSION} = '3'; $ENV{HTTPS_CERT_FILE} = 'spooler-certificate.pem'; $ENV{HTTPS_CERT_PASS} = 'changeit'; $ENV{HTTPS_PKCS12_FILE} = 'spooler-certificate.pfx'; $ENV{HTTPS_PKCS12_PASSWORD} = 'changeit'; #site specific my $rus_address = 'https://grid-data-node06.man.ac.uk:1043/resourceusage/service'; my $records_dir = "/santmp/gmarkets/usage/"; my $jobs_dir = "/santmp/gmarkets/jobs/"; my $records = ""; my @processed_records; my @processed_jobs; my @records; my $count = 0; my $MAX_RECORDS = 10; my $outer_count = 0; my $log_file = "urspool.log"; open LOG, ">>urspool.log"; opendir(RECORDS_DIR, $records_dir); @candidates = readdir RECORDS_DIR; close RECORDS_DIR; foreach $this (@candidates){ if($this =~ /^UR-.*\.xml$/){ my $record = ""; open RECORD, $records_dir.$this; my @inter_record = ; close RECORD; #manipulate the record #remove xml identification shift(@inter_record); @sres = split /-/, $this; $machine_name = $sres[1]; @sres = split /\./, $sres[2]; #/ $local_job_id = $sres[0]; my @inter_job = (); #find the corresponding job file - this is site specific if($machine_name eq "green" || $machine_name eq "wren" || $machine_name eq "fermat"){ $submit_host = "wren"; } else { $submit_host = "newtd"; } $job_file = $jobs_dir."JOB-".$submit_host."-".$local_job_id.".data"; if(-e $job_file){ open JOB, $job_file; @inter_job = ; close JOB; } else { #record with no corresponding job #mark as lost and continue unless(rename $records_dir.$this, $records_dir."lost_".$this){ print LOG "failed to rename $records_dir$this\n"; } next; } #extract X509DistinguishedName, globalJobId, contractId, contractReferenceNumber #from the JOB file foreach $line (@inter_job){ if($line =~ /^#GMARK/){ @line_frag=split " ", $line; if($line_frag[1] eq "-D"){ $distinguished_name = $line_frag[2]; } elsif ($line_frag[1] eq "-N"){ $contract_reference_number = $line_frag[2]; } elsif ($line_frag[1] eq "-C"){ $contract_id = $line_frag[2]; } elsif ($line_frag[1] eq "-J"){ $global_job_id = $line_frag[2]; }; }; }; #add to the list of record files we will attempt to insert push(@processed_records, $this); push(@processed_jobs, $job_file); #add the additional info to the record while(scalar(@inter_record)){ my $temp = shift(@inter_record); #fix up for incorrect namespace in CSAR records - site specific $temp =~ s///g; #add the missing elements $temp=~ s//$distinguished_name/g; $temp=~ s/JSS-UNIQUE-ID/$global_job_id/g; $temp=~ s/Quote 999/$contract_reference_number/g; $temp=~ s/XYZ123/$contract_id/g; if($temp =~ /^/){ $record.="".$machine_name."\n".$submit_host."\n"; } $record.=$temp; } my $parm = SOAP::Data ->name('records') ->value($record) ->attr({xmlns=>'http://www.gmarkets.org/xml/2004/03/resourceusage/types'}); push @records, $parm; $count++; if($count==$MAX_RECORDS){ $start = time; $obj = SOAP::Lite -> uri('http://www.gmarkets.org/xml/2004/03/resourceusage/types') -> proxy($rus_address); $result = $obj -> insertUsageRecordsRequest(@records); if($result){ unless ($result->fault) { my @recordIdList = $result->valueof('//recordListOperationResponse/RecordIdList/RecordId'); $result = $result->result(); %hash = %$result; foreach $element (keys %hash){ print LOG "$element:$hash{$element}\n"; } my $index=0; foreach $id (@recordIdList){; if($id<=0){ if($id==-1){ #permission_denied unless(rename $records_dir.$processed_records[$index], $records_dir."not_authorised_".$processed_records[$index]){ print LOG "failed to rename $records_dir$processed_records[$index]\n"; } } if($id==-2){ #non_existent unless(rename $records_dir.$processed_records[$index], $records_dir."non_existent_".$processed_records[$index]){ print LOG "failed to rename $records_dir$processed_records[$index]\n"; } } if($id==-3){ #invalid unless(rename $records_dir.$processed_records[$index], $records_dir."invalid_".$processed_records[$index]){ print LOG "failed to rename $records_dir$processed_records[$index]\n"; } } if($id==-4){ #duplicate unless(rename $records_dir.$processed_records[$index], $records_dir."duplicate_".$processed_records[$index]){ print LOG "failed to rename $records_dir$processed_records[$index]\n"; } } } else { print LOG $processed_records[$index], " inserted as ", $id, "\n" } $index++; } foreach $file (@processed_records){ unlink $records_dir.$file; } foreach $file (@processed_jobs){ unlink $file; } } else { print LOG "-----------------fault\n"; print LOG join ', ', $result->faultcode, $result->faultstring; print LOG "\n"; print LOG @processed_records; close LOG; exit(1); } } else { print LOG "+++++++++++++++++++++transport error\n"; #stop and wait to be re-scheduled close LOG; exit(2); } $end = time; @records = (); $count=0; @processed_records = (); @processed_jobs = (); } } else { print LOG "skipping $this \n"; } } close LOG; __END__
