ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/SCRAM/src/Cache/Cache.pm
Revision: 1.3
Committed: Fri Mar 11 18:55:28 2005 UTC (20 years, 1 month ago) by sashby
Content type: text/plain
Branch: MAIN
CVS Tags: v102p1, V1_0_1
Changes since 1.2: +8 -3 lines
Log Message:
Fix for problems with files being edited while cache-scanning.

File Contents

# User Rev Content
1 sashby 1.2 #____________________________________________________________________
2     # File: Cache.pm
3     #____________________________________________________________________
4     #
5     # Author: Shaun Ashby <Shaun.Ashby@cern.ch>
6     # (with contribution from Lassi.Tuura@cern.ch)
7     # Update: 2003-11-27 16:45:18+0100
8 sashby 1.3 # Revision: $Id: Cache.pm,v 1.2 2004/12/10 13:41:39 sashby Exp $
9 sashby 1.2 #
10     # Copyright: 2003 (C) Shaun Ashby
11     #
12     #--------------------------------------------------------------------
13     package Cache::Cache;
14     require 5.004;
15    
16     use Exporter;
17     @ISA=qw(Exporter);
18     #
19     sub new()
20     ###############################################################
21     # new #
22     ###############################################################
23     # modified : Thu Nov 27 16:45:27 2003 / SFA #
24     # params : #
25     # : #
26     # function : #
27     # : #
28     ###############################################################
29     {
30     my $proto=shift;
31     my $class=ref($proto) || $proto;
32     my $self=
33     {
34     CACHENAME => "DirCache.db", # Name of global file/dir cache;
35     BFCACHE => {}, # BuildFile cache;
36     DIRCACHE => {}, # Source code cache;
37     STATUS => 0, # Status of cache: 1 => something changed. If so, force save;
38     VERBOSE => 0 # Verbose mode (0/1);
39     };
40    
41     bless $self,$class;
42     return $self;
43     }
44    
45     sub getdir()
46     {
47     my $self=shift;
48     my ($path) = @_;
49     opendir (DIR, $path) || die "$path: cannot read: $!\n";
50 sashby 1.3 # Skip .admin subdirectories too.
51     # Also skip files that look like backup files or files being modified with emacs:
52     my @items = map { "$path/$_" } grep (
53     $_ ne "." && $_ ne ".." &&
54     $_ ne ".admin" && $_ !~ m|\.#*|,
55     readdir(DIR)
56     );
57 sashby 1.2 closedir (DIR);
58     return @items;
59     }
60    
61     sub prune()
62     {
63     my $self=shift;
64     my ($path) = @_;
65     $self->cachestatus(1);
66     return if ! exists $self->{DIRCACHE}->{$path};
67     my (undef, undef, @subs) = @{$self->{DIRCACHE}->{$path}};
68     delete $self->{DIRCACHE}->{$path};
69     foreach my $sub (@subs)
70     {
71     $self->prune($sub);
72     }
73     }
74    
75     sub checktree()
76     {
77     my ($self, $path, $required, $dofiles) = @_;
78     # Check if this path needs to be checked. If it exists, has the same mode
79     # and the same time stamp, it's up to date and doesn't need to be checked.
80     # Otherwise if it is a directory whose time-stamp has changed, rescan it.
81     # If the path has be removed, prune it from the cache. Note that we skip
82     # non-directories unless $dofiles is set. Considering only directories is
83     # dramatically faster.
84     next if ($path =~ /\.admin/); # skip .admin dirs
85    
86     # NB: We stat each path only once ever. The special "_" file handle uses
87     # the results from the last stat we've made. See man perlfunc/stat.
88     if (! stat($path))
89     {
90     die "$path: $!\n" if $required;
91     $self->logmsg("SCRAM: $path: missing: removing from cache\n");
92     $self->prune($path);
93     # Something changed so force write of cache:
94     $self->cachestatus(1);
95     return;
96     }
97    
98     # If the entry in the cache is not the same mode or time, force an update.
99     # Otherwise use the cache as the list of items we need to change.
100     my $cached = $self->{DIRCACHE}->{$path};
101     my @items = ();
102    
103     if (! -d _)
104     {
105     if ($dofiles)
106     {
107     $self->logmsg("SCRAM: $path: updating cache\n");
108     $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9] ];
109     }
110     else
111     {
112     $self->logmsg("SCRAM: $path: not a directory: removing from parent's list\n");
113     my $parent = $path;
114     $parent =~ s|(.*)/[^/]+$|$1|;
115     if ($parent ne $path && exists $self->{DIRCACHE}->{$parent})
116     {
117     my ($mode, $time, @subs) = @{$self->{DIRCACHE}->{$parent}};
118     $self->{DIRCACHE}->{$parent} = [ $mode, $time, grep ($_ ne $path, @subs) ];
119     }
120     $self->cachestatus(1);
121     }
122     }
123     elsif (! $cached || $cached->[0] != (stat(_))[2])
124     {
125     # When a directory is added, this block is activated
126     $self->added_dirs($path); # Store the newly-added dir
127     $self->logmsg("SCRAM: $path: new or changed: pruning and recreating cache\n");
128     $self->prune($path);
129     $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9], @items = $self->getdir($path) ];
130     $required = 1;
131     $self->cachestatus(1);
132     }
133     elsif ($cached->[1] != (stat(_))[9])
134     {
135     # When a subdirectory is removed, this block is activated
136     #
137     # This is a parent directory. We store this as any
138     # update can be taken recursively from this dir:
139     $self->modified_parentdirs($path);
140    
141     $self->logmsg("SCRAM: $path: modified: updating cache\n");
142     # Current subdirs:
143     @items = $self->getdir($path);
144    
145     # Start checking from element number 2:
146     for (my $i = 2; $i <= $#$cached; $i++)
147     {
148     if (! grep($cached->[$i] eq $_, @items))
149     {
150     # Add the removed path to a store for later access
151     # from the project cache. This info is needed to update
152     # the cached data:
153     $self->schedremoval($cached->[$i]);
154     # Remove all child data:
155     $self->clean_cache_recursive($cached->[$i]);
156     }
157     }
158    
159     $self->{DIRCACHE}->{$path} = [ (stat(_))[2, 9], @items ];
160     $required = 1;
161     $self->cachestatus(1);
162     }
163     else
164     {
165     $self->logmsg("SCRAM: $path: valid: using cached directory list\n");
166     (undef, undef, @items) = @{$self->{DIRCACHE}->{$path}};
167     $required = 0;
168     }
169    
170     # Process sub-directories
171     foreach my $item (@items)
172     {
173     $self->checktree($item, $required, $dofiles);
174     }
175     }
176    
177     sub clean_cache_recursive()
178     {
179     my $self=shift;
180     my ($startdir) = @_;
181     my $children = $self->{DIRCACHE}->{$startdir};
182    
183     for (my $i = 2; $i <= $#$children; $i++)
184     {
185     # Remove all children:
186     $self->schedremoval($children->[$i]);
187     $self->clean_cache_recursive($children->[$i]);
188     }
189    
190     delete $self->{DIRCACHE}->{$startdir};
191     return $self;
192     }
193    
194     sub dirtree()
195     {
196     my $self=shift;
197     my ($dir,$dofiles) = @_;
198    
199     # Get the directory tree:
200     $self->checktree($dir, 1, $dofiles);
201     return $self;
202     }
203    
204     sub checkfiles()
205     {
206     my $self=shift;
207     # Scan config dir for top-level data, then start from src:
208     my @scandirs=($ENV{LOCALTOP}."/".$ENV{SCRAM_CONFIGDIR}, $ENV{SCRAM_SOURCEDIR});
209     my $dofiles=1;
210     # Loop over all directories that need scanning (normally just src and config):
211     foreach my $scand (@scandirs)
212     {
213     $self->logmsg("SCRAM: Scanning $scand [dofiles set to ".$dofiles."]\n");
214     # Check the directory tree:
215     $self->dirtree($scand, $dofiles);
216     $dofiles=0;
217     }
218    
219     # Mark everything in the cache old:
220     map { $_->[0] = 0 } values %{$self->{BFCACHE}};
221     map { $_->[0] = 0 } values %{$self->{CONFIGCACHE}};
222    
223     # Remember which directories have buildfiles in them:
224     my %files = map { -f $_ ? ($_ => [ (stat(_))[9] ]) : () }
225     map { "$_/BuildFile" }
226     keys %{$self->{DIRCACHE}};
227    
228     # Get list of files in config dir:
229     my $configcache = {};
230     my %configfiles = map { -f $_ &&
231     $_ =~ m|$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}.*?$|
232     ? ($_ => [ (stat(_))[9] ]) : () } keys %{$self->{DIRCACHE}};
233    
234     # Also add ToolCache.db to the cache: FIXME: should probably use aglob here so
235     # that all SCRAM_ARCHs are taken into account.
236     $configfiles{$ENV{LOCALTOP}."/.SCRAM/".$ENV{SCRAM_ARCH}."/ToolCache.db"} =
237     [ (stat($ENV{LOCALTOP}."/.SCRAM/".$ENV{SCRAM_ARCH}."/ToolCache.db"))[9] ];
238    
239     # Compare or add to config file cache. We need this to be separate so we can tell if a
240     # file affecting our build has been changed:
241     while (my ($path, $vals) = each %configfiles)
242     {
243     if ($self->{CONFIGCACHE}->{$path} && $self->{CONFIGCACHE}->{$path}[1] == $vals->[0])
244     {
245     $configcache->{$path} = $self->{CONFIGCACHE}->{$path};
246     delete $self->{CONFIGCACHE}->{$path};
247     }
248     else
249     {
250     $self->{STATUSCONFIG}=1;
251     $self->logmsg("SCRAM: $path: changed\n");
252     $configcache->{$path} = [ 1, @$vals ];
253     delete $self->{CONFIGCACHE}->{$path};
254     }
255     }
256    
257     # Compare with existing cache: remove from cache what no longer
258     # exists, then check which build files are newer than the cache.
259     my $newcache = {};
260    
261     while (my ($path, $vals) = each %files)
262     {
263     if ($self->{BFCACHE}->{$path} && $self->{BFCACHE}->{$path}[1] == $vals->[0])
264     {
265     $newcache->{$path} = $self->{BFCACHE}->{$path};
266     delete $self->{BFCACHE}->{$path};
267     }
268     else
269     {
270     $self->{STATUSSRC}=1;
271     $self->logmsg("SCRAM: $path: changed\n");
272     $newcache->{$path} = [ 1, @$vals ];
273     delete $self->{BFCACHE}->{$path};
274     }
275     }
276    
277     # If there were BuildFiles that were removed, force update of cache
278     # and remove the BUILDFILEDATA entries:
279     foreach my $path (keys %{$self->{BFCACHE}})
280     {
281     $self->logmsg("SCRAM: $path: removed. Build data will be removed from build cache.\n");
282     $self->cachestatus(1);
283     # Store this so that later, we can tell the BuildDataStore to remove it:
284     $self->schedremoval($path);
285     }
286    
287     # Save the BuildFile cache:
288     delete $self->{BFCACHE};
289     $self->{BFCACHE} = $newcache;
290    
291     # Save the config cache:
292     delete $self->{CONFIGCACHE};
293     $self->{CONFIGCACHE} = $configcache;
294     return $self;
295     }
296    
297     sub dircache()
298     {
299     my $self=shift;
300     # Return the file cache:
301     return $self->{DIRCACHE};
302     }
303    
304     sub added_dirs()
305     {
306     my $self=shift;
307     my ($path) = @_;
308    
309     # If we have a path to add, add it.
310     if ($path)
311     {
312     if (exists($self->{ADDEDDIRS}))
313     {
314     push(@{$self->{ADDEDDIRS}}, $path);
315     }
316     else
317     {
318     $self->{ADDEDDIRS} = [ $path ];
319     }
320     }
321     else
322     {
323     # Otherwise, return the array of added dirs:
324     my @addeddirs = @{$self->{ADDEDDIRS}};
325     delete $self->{ADDEDDIRS};
326     return \@addeddirs;
327     }
328     }
329    
330     sub modified_parentdirs()
331     {
332     my $self=shift;
333     my ($path) = @_;
334    
335     # If we have a path to add, add it.
336     # Don't bother if it's the main source dir as we don't
337     # want to rescan everything from src (that would be silly):
338     if ($path && $path ne $ENV{SCRAM_SOURCEDIR})
339     {
340     if (exists($self->{MODPARENTDIRS}))
341     {
342     push(@{$self->{MODPARENTDIRS}}, $path);
343     }
344     else
345     {
346     $self->{MODPARENTDIRS} = [ $path ];
347     }
348     }
349     else
350     {
351     # Otherwise, return the array of added dirs:
352     my @moddeddirs = @{$self->{MODPARENTDIRS}};
353     delete $self->{MODPARENTDIRS};
354     return \@moddeddirs;
355     }
356     }
357    
358     sub schedremoval()
359     {
360     my $self=shift;
361     my ($d)=@_;
362    
363     if ($d)
364     {
365     if (exists($self->{REMOVEDATA}))
366     {
367     push(@{$self->{REMOVEDATA}},$d);
368     }
369     else
370     {
371     $self->{REMOVEDATA} = [ $d ];
372     }
373     }
374     else
375     {
376     my $remove = [ @{$self->{REMOVEDATA}} ];
377     $self->{REMOVEDATA} = [];
378     return $remove;
379     }
380     }
381    
382     sub filestatus()
383     {
384     my $self=shift;
385     # Here we want to return a true or false value depending on whether
386     # or not a buildfile was changed:
387     return $self->{STATUSSRC};
388     }
389    
390     sub configstatus()
391     {
392     my $self=shift;
393     # Here we want to return a true or false value depending on whether or not a file
394     # in config dir was changed:
395     return $self->{STATUSCONFIG};
396     }
397    
398     sub bf_for_scanning()
399     {
400     my $self=shift;
401     my $MODIFIED = [];
402    
403     $self->{STATUSSRC} = 0;
404    
405     # Return a list of buildfiles to be reread. Note that we only do this
406     # if the status was changed (i.e. don't have to read through the list of BFs to know
407     # whether something changed as the flags STATUSSRC is set as the src tree is checked).
408     # Also we check to see if STATUSCONFIG is true. If so all BuildFiles are marked to be read:
409     if ($self->{STATUSCONFIG})
410     {
411     $self->{STATUSCONFIG} = 0;
412     # Return all the buildfiles since they'll all to be read:
413     return [ keys %{$self->{BFCACHE}} ];
414     }
415     else
416     {
417     # Only return the files that changed:
418     map { ( $self->{BFCACHE}{$_}->[0] == 1 ) && push(@$MODIFIED, $_) } keys %{$self->{BFCACHE}};
419     # Reset the flag:
420     $self->{STATUSCONFIG} = 0;
421     }
422     return $MODIFIED;
423     }
424    
425     sub paths()
426     {
427     my $self=shift;
428     my $paths = {};
429    
430     $self->{ALLDIRS} = [];
431    
432     # Pass over each dir, skipping those that are not wanted and
433     # storing those that are relevant to an array:
434     foreach my $path (keys %{$self->{DIRCACHE}})
435     {
436     if ( ! -d $path && $path != m|$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}.*?$|)
437     {
438     $self->logmsg("SCRAM: $path no longer exists. Clearing from cache.\n");
439     $self->cachestatus(1);
440     delete $self->{DIRCACHE}->{$path};
441     }
442     else
443     {
444     next if $path =~ m|/CVS$|; # Ignore CVS directories.
445     next if $path =~ m|/\.admin$|; # Ignore .admin directories.
446     next if $path =~ m|\Q$ENV{LOCALTOP}/$ENV{SCRAM_CONFIGDIR}\L|;
447     push(@{$self->{ALLDIRS}},$path);
448     }
449     }
450    
451     # Return the array:
452     return $self->{ALLDIRS};
453     }
454    
455     sub verbose()
456     {
457     my $self=shift;
458     # Turn on verbose mode:
459     @_ ? $self->{VERBOSE} = shift
460     : $self->{VERBOSE}
461     }
462    
463     sub cachestatus()
464     {
465     my $self=shift;
466     # Set/return the status of the cache:
467     @_ ? $self->{STATUS} = shift
468     : $self->{STATUS}
469     }
470    
471     sub logmsg()
472     {
473     my $self=shift;
474     # Print a message to STDOUT if VERBOSE is true:
475     print STDERR @_ if $self->verbose();
476     }
477    
478     sub name()
479     {
480     my $self=shift;
481     # Set/return the name of the cache to use:
482     @_ ? $self->{CACHENAME} = shift
483     : $self->{CACHENAME}
484     }
485    
486     1;