ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/SCRAM/src/Cache/Cache.pm
(Generate patch)

Comparing COMP/SCRAM/src/Cache/Cache.pm (file contents):
Revision 1.6 by sashby, Thu Aug 18 15:03:44 2005 UTC vs.
Revision 1.12 by muzaffar, Mon Sep 5 09:01:19 2011 UTC

# Line 35 | Line 35 | package Cache::Cache;
35   require 5.004;
36  
37   use Exporter;
38 + use Utilities::AddDir;
39   @ISA=qw(Exporter);
40   #
41  
42   =item   C<new()>
43  
44 < Create a new Cache::Cache object. The name of the cache is B<DirCache.db> by default.
44 > Create a new Cache::Cache object. The name of the cache is B<DirCache.db.gz> by default.
45  
46   =cut
47  
# Line 59 | Line 60 | sub new()
60     my $class=ref($proto) || $proto;
61     my $self=
62        {
63 <      CACHENAME => "DirCache.db",     # Name of global file/dir cache;
63 >      CACHENAME => "DirCache.db.gz",     # Name of global file/dir cache;
64        BFCACHE => {},                  # BuildFile cache;
65        DIRCACHE => {},                 # Source code cache;
66 +      EXTRASUFFIX => {},              # path with extra suffix;
67        STATUS => 0,                    # Status of cache: 1 => something changed. If so, force save;
68        VERBOSE => 0                    # Verbose mode (0/1);
69        };
# Line 79 | Line 81 | Return a list of directories starting fr
81   sub getdir()
82     {
83     my $self=shift;
84 <   my ($path) = @_;
84 >   my $path=shift;
85 >   my $ignore=shift || 'CVS|\\..*';
86 >   my $match=shift || ".+";
87 >
88     opendir (DIR, $path) || die "$path: cannot read: $!\n";
89     # Skip .admin and CVS subdirectories too.
90     # Also skip files that look like backup files or files being modified with emacs:
91 <   my @items = map { "$path/$_" } grep (
87 <                                        $_ ne "." && $_ ne ".." && $_ ne "CVS" &&
88 <                                        $_ ne ".admin" && $_ !~ m|\.#*|,
89 <                                        readdir(DIR)
90 <                                        );  
91 >   my @items = map { "$path/$_" } grep ((-d "$path/$_") && ($_=~/^($match)$/) && ($_!~/^($ignore)$/),readdir(DIR));
92     closedir (DIR);
93     return @items;
94     }
# Line 101 | Line 102 | Recursively remove directories from the
102   sub prune()
103     {
104     my $self=shift;
105 <   my ($path) = @_;
105 >   my $path = shift;
106 >   my $skipparent = shift || 0;
107 >   my $suffix = shift || "";
108 >   $self->extra_suffix($path,$suffix) if ($suffix);
109 >   if (!$skipparent)
110 >      {
111 >      my $parent = $path;
112 >      $parent =~ s|(.*)/[^/]+$|$1|;
113 >      if ($parent ne $path && exists $self->{DIRCACHE}{$parent})
114 >         {
115 >         my ($time, @subs) = @{$self->{DIRCACHE}{$parent}};
116 >         $self->{DIRCACHE}{$parent} = [ $time, grep ($_ ne $path, @subs) ];
117 >         $self->{ADDEDDIR}{$parent}=1;
118 >         $self->cachestatus(1);
119 >         }
120 >      }
121 >   if (exists $self->{ADDEDDIR}{$path}){delete $self->{ADDEDDIR}{$path};}
122 >   return if ! exists $self->{DIRCACHE}{$path};
123     $self->cachestatus(1);
124 <   return if ! exists $self->{DIRCACHE}->{$path};
125 <   my (undef, undef, @subs) = @{$self->{DIRCACHE}->{$path}};
126 <   delete $self->{DIRCACHE}->{$path};
124 >   foreach my $bf ("$ENV{SCRAM_BUILDFILE}.xml","$ENV{SCRAM_BUILDFILE}")
125 >      {
126 >      if (exists $self->{BFCACHE}{"${path}/${bf}"})
127 >         {
128 >         if (!-f "${path}/${bf}") {$self->{REMOVEDBF}{"${path}/${bf}"}=1;}
129 >         delete $self->{BFCACHE}{"${path}/${bf}"};
130 >         if (exists $self->{ADDEDBF}{"${path}/${bf}"}){delete $self->{ADDEDBF}{"${path}/${bf}"};}
131 >         last;
132 >         }
133 >      }
134 >   if (!-d $path) {$self->{REMOVEDDIR}{$path}=1;}
135 >   my (undef, @subs) = @{$self->{DIRCACHE}{$path}};
136 >   delete $self->{DIRCACHE}{$path};
137     foreach my $sub (@subs)
138        {
139 <      $self->prune($sub);
139 >      $self->prune($sub,1);
140        }
141     }
142  
# Line 121 | Line 149 | directories and their files. Skip all fi
149  
150   sub checktree()
151     {
152 <   my ($self, $path, $required, $dofiles) = @_;
152 >   my ($self, $path, $required) = @_;
153     # Check if this path needs to be checked.  If it exists, has the same mode
154     # and the same time stamp, it's up to date and doesn't need to be checked.
155     # Otherwise if it is a directory whose time-stamp has changed, rescan it.
156     # If the path has be removed, prune it from the cache.  Note that we skip
157     # non-directories unless $dofiles is set.  Considering only directories is
158     # dramatically faster.
131   next if ($path =~ /\.admin/); # skip .admin dirs
132   next if ($path =~ /.*CVS/);
159  
160     # NB: We stat each path only once ever.  The special "_" file handle uses
161     # the results from the last stat we've made.  See man perlfunc/stat.
162     if (! stat($path))
163        {
164        die "$path: $!\n" if $required;
139      $self->logmsg("SCRAM: $path: missing: removing from cache\n");
165        $self->prune($path);
141      # Something changed so force write of cache:
142      $self->cachestatus(1);
166        return;
167        }
168  
169     # If the entry in the cache is not the same mode or time, force an update.
170     # Otherwise use the cache as the list of items we need to change.
171 <   my $cached = $self->{DIRCACHE}->{$path};  
171 >   my $cached = $self->{DIRCACHE}{$path};  
172     my @items = ();
173 +   my $matchdir='[a-zA-Z0-9].+';
174  
175     if (! -d _)
176        {
177 <      if ($dofiles)
178 <         {
155 <         $self->logmsg("SCRAM: $path: updating cache\n");
156 <         $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9] ];
157 <         }
158 <      else
159 <         {
160 <         $self->logmsg("SCRAM: $path: not a directory: removing from parent's list\n");
161 <         my $parent = $path;
162 <         $parent =~ s|(.*)/[^/]+$|$1|;
163 <         if ($parent ne $path && exists $self->{DIRCACHE}->{$parent})
164 <            {
165 <            my ($mode, $time, @subs) = @{$self->{DIRCACHE}->{$parent}};
166 <            $self->{DIRCACHE}->{$parent} = [ $mode, $time, grep ($_ ne $path, @subs) ];
167 <            }
168 <         $self->cachestatus(1);
169 <         }
177 >      $self->prune($path);
178 >      return;
179        }
180 <   elsif (! $cached || $cached->[0] != (stat(_))[2])
180 >   elsif (! $cached)
181        {
182        # When a directory is added, this block is activated
183 <      $self->added_dirs($path); # Store the newly-added dir
184 <      $self->logmsg("SCRAM: $path: new or changed: pruning and recreating cache\n");
176 <      $self->prune($path);
177 <      $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9], @items = $self->getdir($path) ];
183 >      $self->{ADDEDDIR}{$path}=1;
184 >      $self->{DIRCACHE}{$path} = [ (stat(_))[9], @items = $self->getdir($path,'',$matchdir) ];
185        $required = 1;
186        $self->cachestatus(1);
187        }
188 <   elsif ($cached->[1] != (stat(_))[9])
188 >   elsif ($cached->[0] != (stat(_))[9])
189        {
190 +      my $ntime = (stat(_))[9];
191        # When a subdirectory is removed, this block is activated
192        #
193        # This is a parent directory. We store this as any
194        # update can be taken recursively from this dir:
195 <      $self->modified_parentdirs($path);
188 <      
189 <      $self->logmsg("SCRAM: $path: modified: updating cache\n");
195 >      #$self->modified_parentdirs($path);
196        # Current subdirs:
197 <      @items = $self->getdir($path);
198 <      
199 <      # Start checking from element number 2:
200 <      for (my $i = 2; $i <= $#$cached; $i++)
201 <         {
202 <         if (! grep($cached->[$i] eq $_, @items))
197 >      my %curdirs = map { $_ => 1 } $self->getdir($path,'',$matchdir);
198 >      my %olddirs = ();
199 >      for (my $i = 1; $i <= $#$cached; $i++)
200 >         {
201 >         my $d = $cached->[$i];
202 >         $olddirs{$d}=1;
203 >         if (!exists $curdirs{$d})
204              {
205 <            # Add the removed path to a store for later access
199 <            # from the project cache. This info is needed to update
200 <            # the cached data:
201 <            $self->schedremoval($cached->[$i]);
202 <            # Remove all child data:
203 <            $self->clean_cache_recursive($cached->[$i]);
205 >            $self->prune($d,1);
206              }
207 <         }      
207 >         }
208        
209 <      $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9], @items ];
210 <      $required = 1;
209 >      foreach my $d (keys %curdirs)
210 >         {
211 >         if (!exists $olddirs{$d})
212 >            {
213 >            if ($self->extra_suffix($d))
214 >               {
215 >               delete $curdirs{$d};
216 >               }
217 >            }
218 >         }
219 >
220 >      $self->{ADDEDDIR}{$path}=1;
221        $self->cachestatus(1);
222 +      @items = keys %curdirs;
223 +      $required = 0;
224 +      $self->{DIRCACHE}{$path} = [ $ntime, @items ];
225        }
226     else
227        {
228 <      $self->logmsg("SCRAM: $path: valid: using cached directory list\n");
214 <      (undef, undef, @items) = @{$self->{DIRCACHE}->{$path}};
228 >      (undef, @items) = @{$self->{DIRCACHE}{$path}};
229        $required = 0;
230        }
231 +   if (($self->{cachereset}) && (!exists $self->{ADDEDDIR}{$path}))
232 +      {
233 +      $self->{ADDEDDIR}{$path}=1;
234 +      $self->cachestatus(1);
235 +      }
236    
237 +   my $bfcachedir=$ENV{LOCALTOP}."/".$ENV{SCRAM_TMP}."/".$ENV{SCRAM_ARCH}."/cache/bf/${path}";
238 +   my $cbf="${bfcachedir}/$ENV{SCRAM_BUILDFILE}";
239 +   my $bftime=0;
240 +   my $bf="${path}/$ENV{SCRAM_BUILDFILE}";
241 +   foreach my $ext (".xml","")
242 +      {
243 +      my $bfn="$bf$ext";
244 +      if (! stat ($bfn))
245 +         {
246 +         if (exists $self->{BFCACHE}{$bfn})
247 +            {
248 +            $self->{REMOVEDBF}{$bfn}=1;
249 +            delete $self->{BFCACHE}{$bfn};
250 +            Utilities::AddDir::adddir($bfcachedir);
251 +            open(BF,">${cbf}");close(BF);
252 +            $self->cachestatus(1);
253 +            }
254 +         }
255 +      else
256 +         {
257 +         $bftime = (stat(_))[9];
258 +         if ((! exists $self->{BFCACHE}{$bfn}) ||
259 +             ($bftime != $self->{BFCACHE}{$bfn}))
260 +            {
261 +            if ((!-f "${cbf}") || (exists $self->{BFCACHE}{$bfn}))
262 +               {
263 +               Utilities::AddDir::adddir($bfcachedir);
264 +               open(BF,">${cbf}");close(BF);
265 +               }
266 +            $self->{ADDEDBF}{$bfn}=1;
267 +            delete $self->{BFCACHE}{$bf};
268 +            $self->{BFCACHE}{$bfn}=$bftime;
269 +            if ($ext eq ""){$self->{nonxml}+=1;}
270 +            $self->cachestatus(1);
271 +            }
272 +         elsif($self->{cachereset})
273 +            {
274 +            $self->{ADDEDBF}{$bfn}=1;
275 +            if ($ext eq ""){$self->{nonxml}+=1;}
276 +            if (!-f "${cbf}")
277 +               {
278 +               Utilities::AddDir::adddir($bfcachedir);
279 +               open(BF,">${cbf}");close(BF);
280 +               }
281 +            $self->cachestatus(1);
282 +            }
283 +         last;
284 +         }
285 +      }
286 +   if (exists $self->{ExtraDirCache})
287 +      {
288 +      eval {$self->{ExtraDirCache}->DirCache($self,$path);};
289 +      }
290     # Process sub-directories
291     foreach my $item (@items)
292        {
293 <      $self->checktree($item, $required, $dofiles);
293 >      $self->checktree($item, $required);
294        }
295     }
296  
# Line 255 | Line 327 | function just calls checktree().
327   sub dirtree()
328     {
329     my $self=shift;
330 <   my ($dir,$dofiles) = @_;
330 >   my ($dir) = @_;
331  
332     # Get the directory tree:
333 <   $self->checktree($dir, 1, $dofiles);
333 >   $self->checktree($dir, 1);
334     return $self;
335     }
336  
# Line 272 | Line 344 | SCRAM::CMD::build().
344   sub checkfiles()
345     {
346     my $self=shift;
347 +   $self->{cachereset}=shift || 0;
348     # Scan config dir for top-level data, then start from src:
349 <   my @scandirs=($ENV{LOCALTOP}."/".$ENV{SCRAM_CONFIGDIR}, $ENV{SCRAM_SOURCEDIR});
277 <   my $dofiles=1;
349 >   my @scandirs=($ENV{SCRAM_CONFIGDIR}, $ENV{SCRAM_SOURCEDIR});
350     # Loop over all directories that need scanning (normally just src and config):
351 +   $self->{nonxml}=0;
352 +   eval ("use SCRAM::Plugins::DirCache;");
353 +   if(!$@) {$self->{ExtraDirCache} = SCRAM::Plugins::DirCache->new();}
354     foreach my $scand (@scandirs)
355        {
356        $self->logmsg("SCRAM: Scanning $scand [dofiles set to ".$dofiles."]\n");
357        # Check the directory tree:
358 <      $self->dirtree($scand, $dofiles);
284 <      $dofiles=0;
358 >      $self->dirtree($scand);
359        }
360 <  
287 <   # Mark everything in the cache old:
288 <   map { $_->[0] = 0 } values %{$self->{BFCACHE}};
289 <   map { $_->[0] = 0 } values %{$self->{CONFIGCACHE}};
290 <
291 <   # Remember which directories have buildfiles in them:
292 <   my %files = map { -f $_ ? ($_ => [ (stat(_))[9] ]) : () }
293 <               map { "$_/BuildFile" }
294 <               keys %{$self->{DIRCACHE}};
295 <
296 <   # Get list of files in config dir:
297 <   my $configcache = {};
298 <   my %configfiles = map { -f $_ &&
299 <                              $_ =~ m|$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}.*?$|
300 <                              ? ($_ => [ (stat(_))[9] ]) : () } keys %{$self->{DIRCACHE}};
301 <
302 <   # Also add ToolCache.db to the cache: FIXME: should probably use aglob here so
303 <   #                                     that all SCRAM_ARCHs are taken into account.
304 <   $configfiles{$ENV{LOCALTOP}."/.SCRAM/".$ENV{SCRAM_ARCH}."/ToolCache.db"} =
305 <      [ (stat($ENV{LOCALTOP}."/.SCRAM/".$ENV{SCRAM_ARCH}."/ToolCache.db"))[9] ];
306 <  
307 <   # Compare or add to config file cache. We need this to be separate so we can tell if a
308 <   # file affecting our build has been changed:
309 <   while (my ($path, $vals) = each %configfiles)
360 >   if ($self->cachestatus())
361        {
362 <      if ($self->{CONFIGCACHE}->{$path} && $self->{CONFIGCACHE}->{$path}[1] == $vals->[0])
363 <         {
364 <         $configcache->{$path} = $self->{CONFIGCACHE}->{$path};
365 <         delete $self->{CONFIGCACHE}->{$path};
366 <         }
367 <      else
368 <         {
318 <         $self->{STATUSCONFIG}=1;
319 <         $self->logmsg("SCRAM: $path: changed\n");
320 <         $configcache->{$path} = [ 1, @$vals ];
321 <         delete $self->{CONFIGCACHE}->{$path};
362 >      foreach my $bf ("$ENV{SCRAM_BUILDFILE}.xml","$ENV{SCRAM_BUILDFILE}")
363 >         {
364 >         if (exists $self->{BFCACHE}{"$ENV{SCRAM_CONFIGDIR}/${bf}"})
365 >            {
366 >            $self->{ADDEDBF}{"$ENV{SCRAM_CONFIGDIR}/${bf}"}=1;
367 >            last;
368 >            }
369           }
370        }
371 <  
372 <   # Compare with existing cache: remove from cache what no longer
326 <   # exists, then check which build files are newer than the cache.
327 <   my $newcache = {};
328 <
329 <   while (my ($path, $vals) = each %files)
371 >   delete $self->{ExtraDirCache};
372 >   if ($self->{nonxml} > 0)
373        {
374 <      if ($self->{BFCACHE}->{$path} && $self->{BFCACHE}->{$path}[1] == $vals->[0])
332 <         {
333 <         $newcache->{$path} = $self->{BFCACHE}->{$path};
334 <         delete $self->{BFCACHE}->{$path};
335 <         }
336 <      else
337 <         {
338 <         $self->{STATUSSRC}=1;
339 <         $self->logmsg("SCRAM: $path: changed\n");
340 <         $newcache->{$path} = [ 1, @$vals ];
341 <         delete $self->{BFCACHE}->{$path};
342 <         }
374 >      #print STDERR "**** WARNING: ",$self->{nonxml}," non-xml based $ENV{SCRAM_BUILDFILE} were read.\n";
375        }
344
345   # If there were BuildFiles that were removed, force update of cache
346   # and remove the BUILDFILEDATA entries:
347   foreach my $path (keys %{$self->{BFCACHE}})
348      {
349      $self->logmsg("SCRAM: $path: removed. Build data will be removed from build cache.\n");
350      $self->cachestatus(1);      
351      # Store this so that later, we can tell the BuildDataStore to remove it:
352      $self->schedremoval($path);
353      }
354  
355   # Save the BuildFile cache:
356   delete $self->{BFCACHE};
357   $self->{BFCACHE} = $newcache;
358
359   # Save the config cache:
360   delete $self->{CONFIGCACHE};
361   $self->{CONFIGCACHE} = $configcache;
376     return $self;
377     }
378  
# Line 522 | Line 536 | sub bf_for_scanning()
536     {
537     my $self=shift;
538     my $MODIFIED = [];
539 <  
526 <   $self->{STATUSSRC} = 0;
527 <
528 <   # Return a list of buildfiles to be reread. Note that we only do this
529 <   # if the status was changed (i.e. don't have to read through the list of BFs to know
530 <   # whether something changed as the flags STATUSSRC is set as the src tree is checked).
531 <   # Also we check to see if STATUSCONFIG is true. If so all BuildFiles are marked to be read:
532 <   if ($self->{STATUSCONFIG})
533 <      {
534 <      $self->{STATUSCONFIG} = 0;
535 <      # Return all the buildfiles since they'll all to be read:
536 <      return [ keys %{$self->{BFCACHE}} ];
537 <      }
538 <   else
539 <      {
540 <      # Only return the files that changed:
541 <      map { ( $self->{BFCACHE}{$_}->[0] == 1 ) && push(@$MODIFIED, $_) } keys %{$self->{BFCACHE}};
542 <      # Reset the flag:
543 <      $self->{STATUSCONFIG} = 0;
544 <      }
539 >   map { push(@$MODIFIED, $_) } @{$self->{ADDEDBF}};
540     return $MODIFIED;
541     }
542  
# Line 556 | Line 551 | sub paths()
551     my $self=shift;
552     my $paths = {};
553    
554 <   $self->{ALLDIRS} = [];
555 <  
556 <   # Pass over each dir, skipping those that are not wanted and
562 <   # storing those that are relevant to an array:
563 <   foreach my $path (keys %{$self->{DIRCACHE}})
564 <      {
565 <      if ( ! -d $path && $path != m|$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}.*?$|)
566 <         {      
567 <         $self->logmsg("SCRAM: $path no longer exists. Clearing from cache.\n");
568 <         $self->cachestatus(1);
569 <         delete $self->{DIRCACHE}->{$path};
570 <         }
571 <      else
572 <         {
573 <         next if $path =~ m|/CVS$|;     # Ignore CVS directories.
574 <         next if $path =~ m|/\.admin$|; # Ignore .admin directories.
575 <         next if $path =~ m|\Q$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}\L|;
576 <         push(@{$self->{ALLDIRS}},$path);
577 <         }
578 <      }
579 <  
580 <   # Return the array:
581 <   return $self->{ALLDIRS};
554 >   my $ALLDIRS = [];
555 >   map { push(@$ALLDIRS, $_) } keys %{$self->{DIRCACHE}};
556 >   return $ALLDIRS;
557     }
558  
559   =item   C<verbose()>
# Line 627 | Line 602 | sub logmsg()
602   =item   C<name()>
603  
604   Set or return the name of the cache. Normally set
605 < to B<DirCache.db> (and not architecture dependent).
605 > to B<DirCache.db.gz> (and not architecture dependent).
606  
607   =cut
608  
# Line 639 | Line 614 | sub name()
614        : $self->{CACHENAME}
615     }
616  
617 + sub get_data()
618 +   {
619 +     my $self=shift;
620 +     my $type=shift;
621 +     @_ ? $self->{$type} = shift
622 +        : $self->{$type};
623 +   }
624 +
625 + sub extra_suffix()
626 +   {
627 +     my $self=shift;
628 +     my $path=shift;
629 +     @_ ? $self->{EXTRASUFFIX}{$path}=shift
630 +        : exists $self->{EXTRASUFFIX}{$path};
631 +   }
632 +  
633 + sub get_nonxml()
634 +   {
635 +   my $self=shift;
636 +   return $self->{nonxml};
637 +   }
638 +
639   1;
640  
641   =back

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines