ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/SCRAM/src/Cache/Cache.pm
(Generate patch)

Comparing COMP/SCRAM/src/Cache/Cache.pm (file contents):
Revision 1.6 by sashby, Thu Aug 18 15:03:44 2005 UTC vs.
Revision 1.9 by muzaffar, Tue Nov 6 14:13:51 2007 UTC

# Line 35 | Line 35 | package Cache::Cache;
35   require 5.004;
36  
37   use Exporter;
38 + use Utilities::AddDir;
39   @ISA=qw(Exporter);
40   #
41  
# Line 62 | Line 63 | sub new()
63        CACHENAME => "DirCache.db",     # Name of global file/dir cache;
64        BFCACHE => {},                  # BuildFile cache;
65        DIRCACHE => {},                 # Source code cache;
66 +      EXTRASUFFIX => {},              # path with extra suffix;
67        STATUS => 0,                    # Status of cache: 1 => something changed. If so, force save;
68        VERBOSE => 0                    # Verbose mode (0/1);
69        };
# Line 79 | Line 81 | Return a list of directories starting fr
81   sub getdir()
82     {
83     my $self=shift;
84 <   my ($path) = @_;
84 >   my $path=shift;
85 >
86     opendir (DIR, $path) || die "$path: cannot read: $!\n";
87     # Skip .admin and CVS subdirectories too.
88     # Also skip files that look like backup files or files being modified with emacs:
89 <   my @items = map { "$path/$_" } grep (
87 <                                        $_ ne "." && $_ ne ".." && $_ ne "CVS" &&
88 <                                        $_ ne ".admin" && $_ !~ m|\.#*|,
89 <                                        readdir(DIR)
90 <                                        );  
89 >   my @items = map { "$path/$_" } grep ((-d "$path/$_") && ($_!~/^(CVS|\..*)$/),readdir(DIR));
90     closedir (DIR);
91     return @items;
92     }
# Line 101 | Line 100 | Recursively remove directories from the
100   sub prune()
101     {
102     my $self=shift;
103 <   my ($path) = @_;
103 >   my $path = shift;
104 >   my $skipparent = shift || 0;
105 >   my $suffix = shift || "";
106 >   $self->extra_suffix($path,$suffix) if ($suffix);
107 >   if (!$skipparent)
108 >      {
109 >      my $parent = $path;
110 >      $parent =~ s|(.*)/[^/]+$|$1|;
111 >      if ($parent ne $path && exists $self->{DIRCACHE}{$parent})
112 >         {
113 >         my ($time, @subs) = @{$self->{DIRCACHE}{$parent}};
114 >         $self->{DIRCACHE}{$parent} = [ $time, grep ($_ ne $path, @subs) ];
115 >         $self->{ADDEDDIR}{$parent}=1;
116 >         $self->cachestatus(1);
117 >         }
118 >      }
119 >   if (exists $self->{ADDEDDIR}{$path}){delete $self->{ADDEDDIR}{$path};}
120 >   return if ! exists $self->{DIRCACHE}{$path};
121     $self->cachestatus(1);
122 <   return if ! exists $self->{DIRCACHE}->{$path};
123 <   my (undef, undef, @subs) = @{$self->{DIRCACHE}->{$path}};
124 <   delete $self->{DIRCACHE}->{$path};
122 >   foreach my $bf ("$ENV{SCRAM_BUILDFILE}.xml","$ENV{SCRAM_BUILDFILE}")
123 >      {
124 >      if (exists $self->{BFCACHE}{"${path}/${bf}"})
125 >         {
126 >         if (!-f "${path}/${bf}") {$self->{REMOVEDBF}{"${path}/${bf}"}=1;}
127 >         delete $self->{BFCACHE}{"${path}/${bf}"};
128 >         if (exists $self->{ADDEDBF}{"${path}/${bf}"}){delete $self->{ADDEDBF}{"${path}/${bf}"};}
129 >         last;
130 >         }
131 >      }
132 >   if (!-d $path) {$self->{REMOVEDDIR}{$path}=1;}
133 >   my (undef, @subs) = @{$self->{DIRCACHE}{$path}};
134 >   delete $self->{DIRCACHE}{$path};
135     foreach my $sub (@subs)
136        {
137 <      $self->prune($sub);
137 >      $self->prune($sub,1);
138        }
139     }
140  
# Line 121 | Line 147 | directories and their files. Skip all fi
147  
148   sub checktree()
149     {
150 <   my ($self, $path, $required, $dofiles) = @_;
150 >   my ($self, $path, $required) = @_;
151     # Check if this path needs to be checked.  If it exists, has the same mode
152     # and the same time stamp, it's up to date and doesn't need to be checked.
153     # Otherwise if it is a directory whose time-stamp has changed, rescan it.
154     # If the path has be removed, prune it from the cache.  Note that we skip
155     # non-directories unless $dofiles is set.  Considering only directories is
156     # dramatically faster.
131   next if ($path =~ /\.admin/); # skip .admin dirs
132   next if ($path =~ /.*CVS/);
157  
158     # NB: We stat each path only once ever.  The special "_" file handle uses
159     # the results from the last stat we've made.  See man perlfunc/stat.
160     if (! stat($path))
161        {
162        die "$path: $!\n" if $required;
139      $self->logmsg("SCRAM: $path: missing: removing from cache\n");
163        $self->prune($path);
141      # Something changed so force write of cache:
142      $self->cachestatus(1);
164        return;
165        }
166  
167     # If the entry in the cache is not the same mode or time, force an update.
168     # Otherwise use the cache as the list of items we need to change.
169 <   my $cached = $self->{DIRCACHE}->{$path};  
169 >   my $cached = $self->{DIRCACHE}{$path};  
170     my @items = ();
171  
172     if (! -d _)
173        {
174 <      if ($dofiles)
175 <         {
155 <         $self->logmsg("SCRAM: $path: updating cache\n");
156 <         $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9] ];
157 <         }
158 <      else
159 <         {
160 <         $self->logmsg("SCRAM: $path: not a directory: removing from parent's list\n");
161 <         my $parent = $path;
162 <         $parent =~ s|(.*)/[^/]+$|$1|;
163 <         if ($parent ne $path && exists $self->{DIRCACHE}->{$parent})
164 <            {
165 <            my ($mode, $time, @subs) = @{$self->{DIRCACHE}->{$parent}};
166 <            $self->{DIRCACHE}->{$parent} = [ $mode, $time, grep ($_ ne $path, @subs) ];
167 <            }
168 <         $self->cachestatus(1);
169 <         }
174 >      $self->prune($path);
175 >      return;
176        }
177 <   elsif (! $cached || $cached->[0] != (stat(_))[2])
177 >   elsif (! $cached)
178        {
179        # When a directory is added, this block is activated
180 <      $self->added_dirs($path); # Store the newly-added dir
181 <      $self->logmsg("SCRAM: $path: new or changed: pruning and recreating cache\n");
176 <      $self->prune($path);
177 <      $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9], @items = $self->getdir($path) ];
180 >      $self->{ADDEDDIR}{$path}=1;
181 >      $self->{DIRCACHE}{$path} = [ (stat(_))[9], @items = $self->getdir($path) ];
182        $required = 1;
183        $self->cachestatus(1);
184        }
185 <   elsif ($cached->[1] != (stat(_))[9])
185 >   elsif ($cached->[0] != (stat(_))[9])
186        {
187 +      my $ntime = (stat(_))[9];
188        # When a subdirectory is removed, this block is activated
189        #
190        # This is a parent directory. We store this as any
191        # update can be taken recursively from this dir:
192 <      $self->modified_parentdirs($path);
188 <      
189 <      $self->logmsg("SCRAM: $path: modified: updating cache\n");
192 >      #$self->modified_parentdirs($path);
193        # Current subdirs:
194 <      @items = $self->getdir($path);
195 <      
196 <      # Start checking from element number 2:
197 <      for (my $i = 2; $i <= $#$cached; $i++)
198 <         {
199 <         if (! grep($cached->[$i] eq $_, @items))
194 >      my %curdirs = map { $_ => 1 } $self->getdir($path);
195 >      my %olddirs = ();
196 >      for (my $i = 1; $i <= $#$cached; $i++)
197 >         {
198 >         my $d = $cached->[$i];
199 >         $olddirs{$d}=1;
200 >         if (!exists $curdirs{$d})
201              {
202 <            # Add the removed path to a store for later access
199 <            # from the project cache. This info is needed to update
200 <            # the cached data:
201 <            $self->schedremoval($cached->[$i]);
202 <            # Remove all child data:
203 <            $self->clean_cache_recursive($cached->[$i]);
202 >            $self->prune($d,1);
203              }
204 <         }      
204 >         }
205        
206 <      $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9], @items ];
207 <      $required = 1;
206 >      foreach my $d (keys %curdirs)
207 >         {
208 >         if (!exists $olddirs{$d})
209 >            {
210 >            if ($self->extra_suffix($d))
211 >               {
212 >               delete $curdirs{$d};
213 >               }
214 >            }
215 >         }
216 >
217 >      $self->{ADDEDDIR}{$path}=1;
218        $self->cachestatus(1);
219 +      @items = keys %curdirs;
220 +      $required = 0;
221 +      $self->{DIRCACHE}{$path} = [ $ntime, @items ];
222        }
223     else
224        {
225 <      $self->logmsg("SCRAM: $path: valid: using cached directory list\n");
214 <      (undef, undef, @items) = @{$self->{DIRCACHE}->{$path}};
225 >      (undef, @items) = @{$self->{DIRCACHE}{$path}};
226        $required = 0;
227        }
228 +   if (($self->{cachereset}) && (!exists $self->{ADDEDDIR}{$path}))
229 +      {
230 +      $self->{ADDEDDIR}{$path}=1;
231 +      $self->cachestatus(1);
232 +      }
233    
234 +   my $bfcachedir=$ENV{LOCALTOP}."/".$ENV{SCRAM_TMP}."/".$ENV{SCRAM_ARCH}."/cache/bf/${path}";
235 +   my $bftime=0;
236 +   my $bf="${path}/$ENV{SCRAM_BUILDFILE}";
237 +   if (($self->{convertxml}) && (!-f "${bf}.xml") && (-f "$bf"))
238 +       {
239 +       my $fref;
240 +       $self->{nonxml} = $self->{nonxml}+1;
241 +       if (open($fref,">${bf}.xml"))
242 +          {
243 +          print ">> Converting $bf => ${bf}.xml\n";
244 +          $self->{convertxml}->clean();
245 +          my $xml=$self->{convertxml}->convert($bf);
246 +          foreach my $line (@$xml){print $fref "$line\n";}
247 +          close($fref);
248 +          $self->{convertxml}->clean();
249 +          }
250 +       else
251 +          {
252 +          print STDERR "**** WARNING: Can not open file for writing: ${bf}.xml\n";
253 +          }
254 +       }
255 +   foreach my $ext (".xml","")
256 +      {
257 +      my $bfn="$bf$ext";
258 +      if (! stat ($bfn))
259 +         {
260 +         if (exists $self->{BFCACHE}{$bfn})
261 +            {
262 +            $self->{REMOVEDBF}{$bfn}=1;
263 +            delete $self->{BFCACHE}{$bfn};
264 +            AddDir::adddir($bfcachedir);
265 +            open(BF,">${bfcachedir}/$ENV{SCRAM_BUILDFILE}");close(BF);
266 +            $self->cachestatus(1);
267 +            }
268 +         }
269 +      else
270 +         {
271 +         $bftime = (stat(_))[9];
272 +         if ((! exists $self->{BFCACHE}{$bfn}) ||
273 +             ($bftime != $self->{BFCACHE}{$bfn}))
274 +            {
275 +            AddDir::adddir($bfcachedir);
276 +            open(BF,">${bfcachedir}/$ENV{SCRAM_BUILDFILE}");close(BF);
277 +            $self->{ADDEDBF}{$bfn}=1;
278 +            delete $self->{BFCACHE}{"${path}/$ENV{SCRAM_BUILDFILE}"};
279 +            $self->{BFCACHE}{$bfn}=$bftime;
280 +            if ($ext eq ""){$self->{nonxml}+=1;}
281 +            $self->cachestatus(1);
282 +            }
283 +         elsif($self->{cachereset})
284 +            {
285 +            $self->{ADDEDBF}{$bfn}=1;
286 +            if ($ext eq ""){$self->{nonxml}+=1;}
287 +            if (!-f "${bfcachedir}/$ENV{SCRAM_BUILDFILE}")
288 +               {
289 +               AddDir::adddir($bfcachedir);
290 +               open(BF,">${bfcachedir}/$ENV{SCRAM_BUILDFILE}");close(BF);
291 +               }
292 +            $self->cachestatus(1);
293 +            }
294 +         last;
295 +         }
296 +      }
297     # Process sub-directories
298     foreach my $item (@items)
299        {
300 <      $self->checktree($item, $required, $dofiles);
300 >      $self->checktree($item, $required);
301        }
302     }
303  
# Line 255 | Line 334 | function just calls checktree().
334   sub dirtree()
335     {
336     my $self=shift;
337 <   my ($dir,$dofiles) = @_;
337 >   my ($dir) = @_;
338  
339     # Get the directory tree:
340 <   $self->checktree($dir, 1, $dofiles);
340 >   $self->checktree($dir, 1);
341     return $self;
342     }
343  
# Line 272 | Line 351 | SCRAM::CMD::build().
351   sub checkfiles()
352     {
353     my $self=shift;
354 +   $self->{cachereset}=shift || 0;
355 +   $self->{convertxml}=shift || 0;
356     # Scan config dir for top-level data, then start from src:
357 <   my @scandirs=($ENV{LOCALTOP}."/".$ENV{SCRAM_CONFIGDIR}, $ENV{SCRAM_SOURCEDIR});
277 <   my $dofiles=1;
357 >   my @scandirs=($ENV{SCRAM_CONFIGDIR}, $ENV{SCRAM_SOURCEDIR});
358     # Loop over all directories that need scanning (normally just src and config):
359 <   foreach my $scand (@scandirs)
359 >   if ($self->{convertxml})
360        {
361 <      $self->logmsg("SCRAM: Scanning $scand [dofiles set to ".$dofiles."]\n");
362 <      # Check the directory tree:
363 <      $self->dirtree($scand, $dofiles);
364 <      $dofiles=0;
285 <      }
286 <  
287 <   # Mark everything in the cache old:
288 <   map { $_->[0] = 0 } values %{$self->{BFCACHE}};
289 <   map { $_->[0] = 0 } values %{$self->{CONFIGCACHE}};
290 <
291 <   # Remember which directories have buildfiles in them:
292 <   my %files = map { -f $_ ? ($_ => [ (stat(_))[9] ]) : () }
293 <               map { "$_/BuildFile" }
294 <               keys %{$self->{DIRCACHE}};
295 <
296 <   # Get list of files in config dir:
297 <   my $configcache = {};
298 <   my %configfiles = map { -f $_ &&
299 <                              $_ =~ m|$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}.*?$|
300 <                              ? ($_ => [ (stat(_))[9] ]) : () } keys %{$self->{DIRCACHE}};
301 <
302 <   # Also add ToolCache.db to the cache: FIXME: should probably use aglob here so
303 <   #                                     that all SCRAM_ARCHs are taken into account.
304 <   $configfiles{$ENV{LOCALTOP}."/.SCRAM/".$ENV{SCRAM_ARCH}."/ToolCache.db"} =
305 <      [ (stat($ENV{LOCALTOP}."/.SCRAM/".$ENV{SCRAM_ARCH}."/ToolCache.db"))[9] ];
306 <  
307 <   # Compare or add to config file cache. We need this to be separate so we can tell if a
308 <   # file affecting our build has been changed:
309 <   while (my ($path, $vals) = each %configfiles)
310 <      {
311 <      if ($self->{CONFIGCACHE}->{$path} && $self->{CONFIGCACHE}->{$path}[1] == $vals->[0])
312 <         {
313 <         $configcache->{$path} = $self->{CONFIGCACHE}->{$path};
314 <         delete $self->{CONFIGCACHE}->{$path};
361 >      eval ("use SCRAM::Doc2XML");
362 >      if (!$@)
363 >         {
364 >         $self->{convertxml} = SCRAM::Doc2XML->new();
365           }
366        else
367 <         {
368 <         $self->{STATUSCONFIG}=1;
319 <         $self->logmsg("SCRAM: $path: changed\n");
320 <         $configcache->{$path} = [ 1, @$vals ];
321 <         delete $self->{CONFIGCACHE}->{$path};
367 >         {
368 >         print STDERR "**** WARNING: Can not convert $ENV{SCRAM_BUILDFILE} in to XML format. Missing SCRAM::Doc2XML perl module.\n";
369           }
370        }
371 <  
372 <   # Compare with existing cache: remove from cache what no longer
326 <   # exists, then check which build files are newer than the cache.
327 <   my $newcache = {};
328 <
329 <   while (my ($path, $vals) = each %files)
371 >   $self->{nonxml}=0;
372 >   foreach my $scand (@scandirs)
373        {
374 <      if ($self->{BFCACHE}->{$path} && $self->{BFCACHE}->{$path}[1] == $vals->[0])
375 <         {
376 <         $newcache->{$path} = $self->{BFCACHE}->{$path};
377 <         delete $self->{BFCACHE}->{$path};
378 <         }
379 <      else
380 <         {
381 <         $self->{STATUSSRC}=1;
382 <         $self->logmsg("SCRAM: $path: changed\n");
383 <         $newcache->{$path} = [ 1, @$vals ];
384 <         delete $self->{BFCACHE}->{$path};
374 >      $self->logmsg("SCRAM: Scanning $scand [dofiles set to ".$dofiles."]\n");
375 >      # Check the directory tree:
376 >      $self->dirtree($scand);
377 >      }
378 >   if ($self->cachestatus())
379 >      {
380 >      foreach my $bf ("$ENV{SCRAM_BUILDFILE}.xml","$ENV{SCRAM_BUILDFILE}")
381 >         {
382 >         if (exists $self->{BFCACHE}{"$ENV{SCRAM_CONFIGDIR}/${bf}"})
383 >            {
384 >            $self->{ADDEDBF}{"$ENV{SCRAM_CONFIGDIR}/${bf}"}=1;
385 >            last;
386 >            }
387           }
388        }
389 <
345 <   # If there were BuildFiles that were removed, force update of cache
346 <   # and remove the BUILDFILEDATA entries:
347 <   foreach my $path (keys %{$self->{BFCACHE}})
389 >   if ($self->{nonxml} > 0)
390        {
391 <      $self->logmsg("SCRAM: $path: removed. Build data will be removed from build cache.\n");
350 <      $self->cachestatus(1);      
351 <      # Store this so that later, we can tell the BuildDataStore to remove it:
352 <      $self->schedremoval($path);
391 >      print STDERR "**** WARNING: ",$self->{nonxml}," non-xml based $ENV{SCRAM_BUILDFILE} were read.\n";
392        }
354  
355   # Save the BuildFile cache:
356   delete $self->{BFCACHE};
357   $self->{BFCACHE} = $newcache;
358
359   # Save the config cache:
360   delete $self->{CONFIGCACHE};
361   $self->{CONFIGCACHE} = $configcache;
393     return $self;
394     }
395  
# Line 522 | Line 553 | sub bf_for_scanning()
553     {
554     my $self=shift;
555     my $MODIFIED = [];
556 <  
526 <   $self->{STATUSSRC} = 0;
527 <
528 <   # Return a list of buildfiles to be reread. Note that we only do this
529 <   # if the status was changed (i.e. don't have to read through the list of BFs to know
530 <   # whether something changed as the flags STATUSSRC is set as the src tree is checked).
531 <   # Also we check to see if STATUSCONFIG is true. If so all BuildFiles are marked to be read:
532 <   if ($self->{STATUSCONFIG})
533 <      {
534 <      $self->{STATUSCONFIG} = 0;
535 <      # Return all the buildfiles since they'll all to be read:
536 <      return [ keys %{$self->{BFCACHE}} ];
537 <      }
538 <   else
539 <      {
540 <      # Only return the files that changed:
541 <      map { ( $self->{BFCACHE}{$_}->[0] == 1 ) && push(@$MODIFIED, $_) } keys %{$self->{BFCACHE}};
542 <      # Reset the flag:
543 <      $self->{STATUSCONFIG} = 0;
544 <      }
556 >   map { push(@$MODIFIED, $_) } @{$self->{ADDEDBF}};
557     return $MODIFIED;
558     }
559  
# Line 556 | Line 568 | sub paths()
568     my $self=shift;
569     my $paths = {};
570    
571 <   $self->{ALLDIRS} = [];
572 <  
573 <   # Pass over each dir, skipping those that are not wanted and
562 <   # storing those that are relevant to an array:
563 <   foreach my $path (keys %{$self->{DIRCACHE}})
564 <      {
565 <      if ( ! -d $path && $path != m|$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}.*?$|)
566 <         {      
567 <         $self->logmsg("SCRAM: $path no longer exists. Clearing from cache.\n");
568 <         $self->cachestatus(1);
569 <         delete $self->{DIRCACHE}->{$path};
570 <         }
571 <      else
572 <         {
573 <         next if $path =~ m|/CVS$|;     # Ignore CVS directories.
574 <         next if $path =~ m|/\.admin$|; # Ignore .admin directories.
575 <         next if $path =~ m|\Q$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}\L|;
576 <         push(@{$self->{ALLDIRS}},$path);
577 <         }
578 <      }
579 <  
580 <   # Return the array:
581 <   return $self->{ALLDIRS};
571 >   my $ALLDIRS = [];
572 >   map { push(@$ALLDIRS, $_) } keys %{$self->{DIRCACHE}};
573 >   return $ALLDIRS;
574     }
575  
576   =item   C<verbose()>
# Line 639 | Line 631 | sub name()
631        : $self->{CACHENAME}
632     }
633  
634 + sub get_data()
635 +   {
636 +     my $self=shift;
637 +     my $type=shift;
638 +     @_ ? $self->{$type} = shift
639 +        : $self->{$type};
640 +   }
641 +
642 + sub extra_suffix()
643 +   {
644 +     my $self=shift;
645 +     my $path=shift;
646 +     @_ ? $self->{EXTRASUFFIX}{$path}=shift
647 +        : exists $self->{EXTRASUFFIX}{$path};
648 +   }
649 +  
650 + sub get_nonxml()
651 +   {
652 +   my $self=shift;
653 +   return $self->{nonxml};
654 +   }
655 +
656   1;
657  
658   =back

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines