ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/SCRAM/src/Cache/Cache.pm
(Generate patch)

Comparing COMP/SCRAM/src/Cache/Cache.pm (file contents):
Revision 1.6 by sashby, Thu Aug 18 15:03:44 2005 UTC vs.
Revision 1.14 by muzaffar, Tue Oct 18 14:59:28 2011 UTC

# Line 4 | Line 4
4   #  
5   # Author: Shaun Ashby <Shaun.Ashby@cern.ch>
6   #         (with contribution from Lassi.Tuura@cern.ch)
7 # Update: 2003-11-27 16:45:18+0100
8 # Revision: $Id$
9 #
7   # Copyright: 2003 (C) Shaun Ashby
8   #
9   #--------------------------------------------------------------------
# Line 35 | Line 32 | package Cache::Cache;
32   require 5.004;
33  
34   use Exporter;
35 + use Utilities::AddDir;
36   @ISA=qw(Exporter);
37   #
38  
39   =item   C<new()>
40  
41 < Create a new Cache::Cache object. The name of the cache is B<DirCache.db> by default.
41 > Create a new Cache::Cache object. The name of the cache is B<DirCache.db.gz> by default.
42  
43   =cut
44  
# Line 59 | Line 57 | sub new()
57     my $class=ref($proto) || $proto;
58     my $self=
59        {
60 <      CACHENAME => "DirCache.db",     # Name of global file/dir cache;
60 >      CACHENAME => "DirCache.db.gz",     # Name of global file/dir cache;
61        BFCACHE => {},                  # BuildFile cache;
62        DIRCACHE => {},                 # Source code cache;
63 +      EXTRASUFFIX => {},              # path with extra suffix;
64        STATUS => 0,                    # Status of cache: 1 => something changed. If so, force save;
65        VERBOSE => 0                    # Verbose mode (0/1);
66        };
# Line 79 | Line 78 | Return a list of directories starting fr
78   sub getdir()
79     {
80     my $self=shift;
81 <   my ($path) = @_;
81 >   my $path=shift;
82 >   my $ignore=shift || 'CVS|\\..*';
83 >   my $match=shift || ".+";
84 >
85     opendir (DIR, $path) || die "$path: cannot read: $!\n";
86     # Skip .admin and CVS subdirectories too.
87     # Also skip files that look like backup files or files being modified with emacs:
88 <   my @items = map { "$path/$_" } grep (
87 <                                        $_ ne "." && $_ ne ".." && $_ ne "CVS" &&
88 <                                        $_ ne ".admin" && $_ !~ m|\.#*|,
89 <                                        readdir(DIR)
90 <                                        );  
88 >   my @items = map { "$path/$_" } grep ((-d "$path/$_") && ($_=~/^($match)$/) && ($_!~/^($ignore)$/),readdir(DIR));
89     closedir (DIR);
90     return @items;
91     }
# Line 101 | Line 99 | Recursively remove directories from the
99   sub prune()
100     {
101     my $self=shift;
102 <   my ($path) = @_;
102 >   my $path = shift;
103 >   my $skipparent = shift || 0;
104 >   my $suffix = shift || "";
105 >   $self->extra_suffix($path,$suffix) if ($suffix);
106 >   if (!$skipparent)
107 >      {
108 >      my $parent = $path;
109 >      $parent =~ s|(.*)/[^/]+$|$1|;
110 >      if ($parent ne $path && exists $self->{DIRCACHE}{$parent})
111 >         {
112 >         my ($time, @subs) = @{$self->{DIRCACHE}{$parent}};
113 >         $self->{DIRCACHE}{$parent} = [ $time, grep ($_ ne $path, @subs) ];
114 >         $self->{ADDEDDIR}{$parent}=1;
115 >         $self->cachestatus(1);
116 >         }
117 >      }
118 >   if (exists $self->{ADDEDDIR}{$path}){delete $self->{ADDEDDIR}{$path};}
119 >   return if ! exists $self->{DIRCACHE}{$path};
120     $self->cachestatus(1);
121 <   return if ! exists $self->{DIRCACHE}->{$path};
122 <   my (undef, undef, @subs) = @{$self->{DIRCACHE}->{$path}};
123 <   delete $self->{DIRCACHE}->{$path};
121 >   foreach my $bf ("$ENV{SCRAM_BUILDFILE}.xml","$ENV{SCRAM_BUILDFILE}")
122 >      {
123 >      if (exists $self->{BFCACHE}{"${path}/${bf}"})
124 >         {
125 >         if (!-f "${path}/${bf}") {$self->{REMOVEDBF}{"${path}/${bf}"}=1;}
126 >         delete $self->{BFCACHE}{"${path}/${bf}"};
127 >         if (exists $self->{ADDEDBF}{"${path}/${bf}"}){delete $self->{ADDEDBF}{"${path}/${bf}"};}
128 >         last;
129 >         }
130 >      }
131 >   if (!-d $path) {$self->{REMOVEDDIR}{$path}=1;}
132 >   my (undef, @subs) = @{$self->{DIRCACHE}{$path}};
133 >   delete $self->{DIRCACHE}{$path};
134     foreach my $sub (@subs)
135        {
136 <      $self->prune($sub);
136 >      $self->prune($sub,1);
137        }
138     }
139  
# Line 121 | Line 146 | directories and their files. Skip all fi
146  
147   sub checktree()
148     {
149 <   my ($self, $path, $required, $dofiles) = @_;
149 >   my ($self, $path, $required) = @_;
150     # Check if this path needs to be checked.  If it exists, has the same mode
151     # and the same time stamp, it's up to date and doesn't need to be checked.
152     # Otherwise if it is a directory whose time-stamp has changed, rescan it.
153     # If the path has be removed, prune it from the cache.  Note that we skip
154     # non-directories unless $dofiles is set.  Considering only directories is
155     # dramatically faster.
131   next if ($path =~ /\.admin/); # skip .admin dirs
132   next if ($path =~ /.*CVS/);
156  
157     # NB: We stat each path only once ever.  The special "_" file handle uses
158     # the results from the last stat we've made.  See man perlfunc/stat.
159     if (! stat($path))
160        {
161        die "$path: $!\n" if $required;
139      $self->logmsg("SCRAM: $path: missing: removing from cache\n");
162        $self->prune($path);
141      # Something changed so force write of cache:
142      $self->cachestatus(1);
163        return;
164        }
165  
166     # If the entry in the cache is not the same mode or time, force an update.
167     # Otherwise use the cache as the list of items we need to change.
168 <   my $cached = $self->{DIRCACHE}->{$path};  
168 >   my $cached = $self->{DIRCACHE}{$path};  
169     my @items = ();
170 +   my $matchdir='[a-zA-Z0-9][a-zA-Z0-9-_]*';
171  
172     if (! -d _)
173        {
174 <      if ($dofiles)
175 <         {
155 <         $self->logmsg("SCRAM: $path: updating cache\n");
156 <         $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9] ];
157 <         }
158 <      else
159 <         {
160 <         $self->logmsg("SCRAM: $path: not a directory: removing from parent's list\n");
161 <         my $parent = $path;
162 <         $parent =~ s|(.*)/[^/]+$|$1|;
163 <         if ($parent ne $path && exists $self->{DIRCACHE}->{$parent})
164 <            {
165 <            my ($mode, $time, @subs) = @{$self->{DIRCACHE}->{$parent}};
166 <            $self->{DIRCACHE}->{$parent} = [ $mode, $time, grep ($_ ne $path, @subs) ];
167 <            }
168 <         $self->cachestatus(1);
169 <         }
174 >      $self->prune($path);
175 >      return;
176        }
177 <   elsif (! $cached || $cached->[0] != (stat(_))[2])
177 >   elsif (! $cached)
178        {
179        # When a directory is added, this block is activated
180 <      $self->added_dirs($path); # Store the newly-added dir
181 <      $self->logmsg("SCRAM: $path: new or changed: pruning and recreating cache\n");
176 <      $self->prune($path);
177 <      $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9], @items = $self->getdir($path) ];
180 >      $self->{ADDEDDIR}{$path}=1;
181 >      $self->{DIRCACHE}{$path} = [ (stat(_))[9], @items = $self->getdir($path,'',$matchdir) ];
182        $required = 1;
183        $self->cachestatus(1);
184        }
185 <   elsif ($cached->[1] != (stat(_))[9])
185 >   elsif ($cached->[0] != (stat(_))[9])
186        {
187 +      my $ntime = (stat(_))[9];
188        # When a subdirectory is removed, this block is activated
189        #
190        # This is a parent directory. We store this as any
191        # update can be taken recursively from this dir:
192 <      $self->modified_parentdirs($path);
188 <      
189 <      $self->logmsg("SCRAM: $path: modified: updating cache\n");
192 >      #$self->modified_parentdirs($path);
193        # Current subdirs:
194 <      @items = $self->getdir($path);
195 <      
196 <      # Start checking from element number 2:
197 <      for (my $i = 2; $i <= $#$cached; $i++)
198 <         {
199 <         if (! grep($cached->[$i] eq $_, @items))
194 >      my %curdirs = map { $_ => 1 } $self->getdir($path,'',$matchdir);
195 >      my %olddirs = ();
196 >      for (my $i = 1; $i <= $#$cached; $i++)
197 >         {
198 >         my $d = $cached->[$i];
199 >         $olddirs{$d}=1;
200 >         if (!exists $curdirs{$d})
201              {
202 <            # Add the removed path to a store for later access
199 <            # from the project cache. This info is needed to update
200 <            # the cached data:
201 <            $self->schedremoval($cached->[$i]);
202 <            # Remove all child data:
203 <            $self->clean_cache_recursive($cached->[$i]);
202 >            $self->prune($d,1);
203              }
204 <         }      
204 >         }
205        
206 <      $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9], @items ];
207 <      $required = 1;
206 >      foreach my $d (keys %curdirs)
207 >         {
208 >         if (!exists $olddirs{$d})
209 >            {
210 >            if ($self->extra_suffix($d))
211 >               {
212 >               delete $curdirs{$d};
213 >               }
214 >            }
215 >         }
216 >
217 >      $self->{ADDEDDIR}{$path}=1;
218        $self->cachestatus(1);
219 +      @items = keys %curdirs;
220 +      $required = 0;
221 +      $self->{DIRCACHE}{$path} = [ $ntime, @items ];
222        }
223     else
224        {
225 <      $self->logmsg("SCRAM: $path: valid: using cached directory list\n");
214 <      (undef, undef, @items) = @{$self->{DIRCACHE}->{$path}};
225 >      (undef, @items) = @{$self->{DIRCACHE}{$path}};
226        $required = 0;
227        }
228 +   if (($self->{cachereset}) && (!exists $self->{ADDEDDIR}{$path}))
229 +      {
230 +      $self->{ADDEDDIR}{$path}=1;
231 +      $self->cachestatus(1);
232 +      }
233    
234 +   my $bfcachedir=$ENV{LOCALTOP}."/".$ENV{SCRAM_TMP}."/".$ENV{SCRAM_ARCH}."/cache/bf/${path}";
235 +   my $cbf="${bfcachedir}/$ENV{SCRAM_BUILDFILE}";
236 +   my $bftime=0;
237 +   my $bf="${path}/$ENV{SCRAM_BUILDFILE}";
238 +   foreach my $ext (".xml","")
239 +      {
240 +      my $bfn="$bf$ext";
241 +      if (! stat ($bfn))
242 +         {
243 +         if (exists $self->{BFCACHE}{$bfn})
244 +            {
245 +            $self->{REMOVEDBF}{$bfn}=1;
246 +            delete $self->{BFCACHE}{$bfn};
247 +            Utilities::AddDir::adddir($bfcachedir);
248 +            open(BF,">${cbf}");close(BF);
249 +            $self->cachestatus(1);
250 +            }
251 +         }
252 +      else
253 +         {
254 +         $bftime = (stat(_))[9];
255 +         if ((! exists $self->{BFCACHE}{$bfn}) ||
256 +             ($bftime != $self->{BFCACHE}{$bfn}))
257 +            {
258 +            if ((!-f "${cbf}") || (exists $self->{BFCACHE}{$bfn}))
259 +               {
260 +               Utilities::AddDir::adddir($bfcachedir);
261 +               open(BF,">${cbf}");close(BF);
262 +               }
263 +            $self->{ADDEDBF}{$bfn}=1;
264 +            delete $self->{BFCACHE}{$bf};
265 +            $self->{BFCACHE}{$bfn}=$bftime;
266 +            if ($ext eq ""){$self->{nonxml}+=1;}
267 +            $self->cachestatus(1);
268 +            }
269 +         elsif($self->{cachereset})
270 +            {
271 +            $self->{ADDEDBF}{$bfn}=1;
272 +            if ($ext eq ""){$self->{nonxml}+=1;}
273 +            if (!-f "${cbf}")
274 +               {
275 +               Utilities::AddDir::adddir($bfcachedir);
276 +               open(BF,">${cbf}");close(BF);
277 +               }
278 +            $self->cachestatus(1);
279 +            }
280 +         last;
281 +         }
282 +      }
283 +   if (exists $self->{ExtraDirCache})
284 +      {
285 +      eval {$self->{ExtraDirCache}->DirCache($self,$path);};
286 +      }
287     # Process sub-directories
288     foreach my $item (@items)
289        {
290 <      $self->checktree($item, $required, $dofiles);
290 >      $self->checktree($item, $required);
291        }
292     }
293  
# Line 255 | Line 324 | function just calls checktree().
324   sub dirtree()
325     {
326     my $self=shift;
327 <   my ($dir,$dofiles) = @_;
327 >   my ($dir) = @_;
328  
329     # Get the directory tree:
330 <   $self->checktree($dir, 1, $dofiles);
330 >   $self->checktree($dir, 1);
331     return $self;
332     }
333  
# Line 272 | Line 341 | SCRAM::CMD::build().
341   sub checkfiles()
342     {
343     my $self=shift;
344 +   $self->{cachereset}=shift || 0;
345     # Scan config dir for top-level data, then start from src:
346 <   my @scandirs=($ENV{LOCALTOP}."/".$ENV{SCRAM_CONFIGDIR}, $ENV{SCRAM_SOURCEDIR});
277 <   my $dofiles=1;
346 >   my @scandirs=($ENV{SCRAM_CONFIGDIR}, $ENV{SCRAM_SOURCEDIR});
347     # Loop over all directories that need scanning (normally just src and config):
348 +   $self->{nonxml}=0;
349 +   eval ("use SCRAM::Plugins::DirCache;");
350 +   if(!$@) {$self->{ExtraDirCache} = SCRAM::Plugins::DirCache->new();}
351     foreach my $scand (@scandirs)
352        {
353        $self->logmsg("SCRAM: Scanning $scand [dofiles set to ".$dofiles."]\n");
354        # Check the directory tree:
355 <      $self->dirtree($scand, $dofiles);
284 <      $dofiles=0;
355 >      $self->dirtree($scand);
356        }
357 <  
287 <   # Mark everything in the cache old:
288 <   map { $_->[0] = 0 } values %{$self->{BFCACHE}};
289 <   map { $_->[0] = 0 } values %{$self->{CONFIGCACHE}};
290 <
291 <   # Remember which directories have buildfiles in them:
292 <   my %files = map { -f $_ ? ($_ => [ (stat(_))[9] ]) : () }
293 <               map { "$_/BuildFile" }
294 <               keys %{$self->{DIRCACHE}};
295 <
296 <   # Get list of files in config dir:
297 <   my $configcache = {};
298 <   my %configfiles = map { -f $_ &&
299 <                              $_ =~ m|$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}.*?$|
300 <                              ? ($_ => [ (stat(_))[9] ]) : () } keys %{$self->{DIRCACHE}};
301 <
302 <   # Also add ToolCache.db to the cache: FIXME: should probably use aglob here so
303 <   #                                     that all SCRAM_ARCHs are taken into account.
304 <   $configfiles{$ENV{LOCALTOP}."/.SCRAM/".$ENV{SCRAM_ARCH}."/ToolCache.db"} =
305 <      [ (stat($ENV{LOCALTOP}."/.SCRAM/".$ENV{SCRAM_ARCH}."/ToolCache.db"))[9] ];
306 <  
307 <   # Compare or add to config file cache. We need this to be separate so we can tell if a
308 <   # file affecting our build has been changed:
309 <   while (my ($path, $vals) = each %configfiles)
357 >   if ($self->cachestatus())
358        {
359 <      if ($self->{CONFIGCACHE}->{$path} && $self->{CONFIGCACHE}->{$path}[1] == $vals->[0])
360 <         {
361 <         $configcache->{$path} = $self->{CONFIGCACHE}->{$path};
362 <         delete $self->{CONFIGCACHE}->{$path};
363 <         }
364 <      else
365 <         {
318 <         $self->{STATUSCONFIG}=1;
319 <         $self->logmsg("SCRAM: $path: changed\n");
320 <         $configcache->{$path} = [ 1, @$vals ];
321 <         delete $self->{CONFIGCACHE}->{$path};
359 >      foreach my $bf ("$ENV{SCRAM_BUILDFILE}.xml","$ENV{SCRAM_BUILDFILE}")
360 >         {
361 >         if (exists $self->{BFCACHE}{"$ENV{SCRAM_CONFIGDIR}/${bf}"})
362 >            {
363 >            $self->{ADDEDBF}{"$ENV{SCRAM_CONFIGDIR}/${bf}"}=1;
364 >            last;
365 >            }
366           }
367        }
368 <  
369 <   # Compare with existing cache: remove from cache what no longer
326 <   # exists, then check which build files are newer than the cache.
327 <   my $newcache = {};
328 <
329 <   while (my ($path, $vals) = each %files)
368 >   delete $self->{ExtraDirCache};
369 >   if ($self->{nonxml} > 0)
370        {
371 <      if ($self->{BFCACHE}->{$path} && $self->{BFCACHE}->{$path}[1] == $vals->[0])
332 <         {
333 <         $newcache->{$path} = $self->{BFCACHE}->{$path};
334 <         delete $self->{BFCACHE}->{$path};
335 <         }
336 <      else
337 <         {
338 <         $self->{STATUSSRC}=1;
339 <         $self->logmsg("SCRAM: $path: changed\n");
340 <         $newcache->{$path} = [ 1, @$vals ];
341 <         delete $self->{BFCACHE}->{$path};
342 <         }
343 <      }
344 <
345 <   # If there were BuildFiles that were removed, force update of cache
346 <   # and remove the BUILDFILEDATA entries:
347 <   foreach my $path (keys %{$self->{BFCACHE}})
348 <      {
349 <      $self->logmsg("SCRAM: $path: removed. Build data will be removed from build cache.\n");
350 <      $self->cachestatus(1);      
351 <      # Store this so that later, we can tell the BuildDataStore to remove it:
352 <      $self->schedremoval($path);
371 >      #print STDERR "**** WARNING: ",$self->{nonxml}," non-xml based $ENV{SCRAM_BUILDFILE} were read.\n";
372        }
354  
355   # Save the BuildFile cache:
356   delete $self->{BFCACHE};
357   $self->{BFCACHE} = $newcache;
358
359   # Save the config cache:
360   delete $self->{CONFIGCACHE};
361   $self->{CONFIGCACHE} = $configcache;
373     return $self;
374     }
375  
# Line 522 | Line 533 | sub bf_for_scanning()
533     {
534     my $self=shift;
535     my $MODIFIED = [];
536 <  
526 <   $self->{STATUSSRC} = 0;
527 <
528 <   # Return a list of buildfiles to be reread. Note that we only do this
529 <   # if the status was changed (i.e. don't have to read through the list of BFs to know
530 <   # whether something changed as the flags STATUSSRC is set as the src tree is checked).
531 <   # Also we check to see if STATUSCONFIG is true. If so all BuildFiles are marked to be read:
532 <   if ($self->{STATUSCONFIG})
533 <      {
534 <      $self->{STATUSCONFIG} = 0;
535 <      # Return all the buildfiles since they'll all to be read:
536 <      return [ keys %{$self->{BFCACHE}} ];
537 <      }
538 <   else
539 <      {
540 <      # Only return the files that changed:
541 <      map { ( $self->{BFCACHE}{$_}->[0] == 1 ) && push(@$MODIFIED, $_) } keys %{$self->{BFCACHE}};
542 <      # Reset the flag:
543 <      $self->{STATUSCONFIG} = 0;
544 <      }
536 >   map { push(@$MODIFIED, $_) } @{$self->{ADDEDBF}};
537     return $MODIFIED;
538     }
539  
# Line 556 | Line 548 | sub paths()
548     my $self=shift;
549     my $paths = {};
550    
551 <   $self->{ALLDIRS} = [];
552 <  
553 <   # Pass over each dir, skipping those that are not wanted and
562 <   # storing those that are relevant to an array:
563 <   foreach my $path (keys %{$self->{DIRCACHE}})
564 <      {
565 <      if ( ! -d $path && $path != m|$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}.*?$|)
566 <         {      
567 <         $self->logmsg("SCRAM: $path no longer exists. Clearing from cache.\n");
568 <         $self->cachestatus(1);
569 <         delete $self->{DIRCACHE}->{$path};
570 <         }
571 <      else
572 <         {
573 <         next if $path =~ m|/CVS$|;     # Ignore CVS directories.
574 <         next if $path =~ m|/\.admin$|; # Ignore .admin directories.
575 <         next if $path =~ m|\Q$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}\L|;
576 <         push(@{$self->{ALLDIRS}},$path);
577 <         }
578 <      }
579 <  
580 <   # Return the array:
581 <   return $self->{ALLDIRS};
551 >   my $ALLDIRS = [];
552 >   map { push(@$ALLDIRS, $_) } keys %{$self->{DIRCACHE}};
553 >   return $ALLDIRS;
554     }
555  
556   =item   C<verbose()>
# Line 627 | Line 599 | sub logmsg()
599   =item   C<name()>
600  
601   Set or return the name of the cache. Normally set
602 < to B<DirCache.db> (and not architecture dependent).
602 > to B<DirCache.db.gz> (and not architecture dependent).
603  
604   =cut
605  
# Line 639 | Line 611 | sub name()
611        : $self->{CACHENAME}
612     }
613  
614 + sub get_data()
615 +   {
616 +     my $self=shift;
617 +     my $type=shift;
618 +     @_ ? $self->{$type} = shift
619 +        : $self->{$type};
620 +   }
621 +
622 + sub extra_suffix()
623 +   {
624 +     my $self=shift;
625 +     my $path=shift;
626 +     @_ ? $self->{EXTRASUFFIX}{$path}=shift
627 +        : exists $self->{EXTRASUFFIX}{$path};
628 +   }
629 +  
630 + sub get_nonxml()
631 +   {
632 +   my $self=shift;
633 +   return $self->{nonxml};
634 +   }
635 +
636   1;
637  
638   =back

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines