Mail Index


[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

Re: [ApacheGallery] directory generating slowness



On Thu, Dec 09, 2004 at 08:34:13AM -0500, Doug Alcorn wrote:
> I'm bothered by the amount of time that A::G takes to generate
> directory listings.  I've looked briefly at that code.  It seems like
> it's doing a lot of fast things in sequence that ends up being overall
> slow.

Here's my first patch.  It yanks all the stuff out of the handler for
generating the directory page and makes a sub generate_directory.  The
actual page generation code is unchanged except that
generate_directory dumps it's output to cache_dir($r, 0) .
"/index.html".  The the handler sub checks the timestamp on the
.cache/index.html versus the timestamp of the image directory and
either serves up the cached index or generates a new one.  I made a
half-hearted attempt to determine when A::G was instantiated, but was
unable to make it work like it should.

You can use this patch or /dev/null it.

--- Apache-Gallery-0.9.1/lib/Apache/Gallery.pm	Sat Sep 11 17:53:05 2004
+++ Apache-Gallery-devel/lib/Apache/Gallery.pm	Thu Dec  9 11:56:58 2004
@@ -5,9 +5,10 @@
 
 use strict;
 
-use vars qw($VERSION);
+use vars qw($VERSION $GALLERY_STARTTIME);
 
 $VERSION = "0.9.1";
+$GALLERY_STARTTIME = "";
 
 BEGIN {
 
@@ -39,10 +40,10 @@
 		Apache::Constants->import('OK','DECLINED','FORBIDDEN','NOT_FOUND');
 
 	}
-
+        $::GALLERY_STARTTIME = time;
 }
 
 use Image::Info qw(image_info);
 use Image::Size qw(imgsize);
 use Image::Imlib2;
 use Text::Template qw(fill_in_file);
@@ -182,244 +192,30 @@
 			}
 		}
 
-		my $tpl_dir = $r->dir_config('GalleryTemplateDir');
-
-		my %tpl_vars = (layout    => "$tpl_dir/layout.tpl",
-				index     => "$tpl_dir/index.tpl",
-				directory => "$tpl_dir/directory.tpl",
-				picture   => "$tpl_dir/picture.tpl",
-				file      => "$tpl_dir/file.tpl",
-				comment   => "$tpl_dir/dircomment.tpl",
-				nocomment => "$tpl_dir/nodircomment.tpl",
-		);
-
-		$tpl_vars{TITLE} = "Index of: $uri";
-		$tpl_vars{META} = " ";
-
-		unless (opendir (DIR, $filename)) {
-			show_error ($r, 500, $!, "Unable to access directory $filename: $!");
+                my $cacheindex = cache_dir($r, 0) . "/index.html";
+                my $index;
+                my $save_index = 0;
+                if ( -f $cacheindex && -r _ 
+                     && stat($cacheindex)->mtime() > stat(cache_dir($r, 0))->mtime() ) {
+#                     && stat($cacheindex)->mtime() > $::GALLERY_STARTTIME) {
+                    unless (open CACHE, $cacheindex) {
+                        $r->warn("Can't read '$cacheindex', $!\n");
+			show_error($r, 404, "Unable to access file", "Unable to access directory $cacheindex");
 			return $::MP2 ? Apache::OK() : Apache::Constants::OK();
-		}
-
-		$tpl_vars{MENU} = generate_menu($r);
-
-		$tpl_vars{FORM_BEGIN} = $select_mode?'<form method="post">':'';
-		$tpl_vars{FORM_END}   = $select_mode?'</form>':'';
-
-		# Read, sort, and filter files
-		my @files = grep { !/^\./ && -f "$filename/$_" } readdir (DIR);
-
-		@files=gallerysort($r, @files);
-
-		my @downloadable_files;
-
-		if (@files) {
-			# Remove unwanted files from list
-			my @new_files = ();
-			foreach my $picture (@files) {
-
-				my $file = $topdir."/".$picture;
-
-				if ($file =~ /$doc_pattern/i) {
-					push (@downloadable_files, $picture);
-					
-				}
-
-				if ($file =~ /$img_pattern/i) {
-					push (@new_files, $picture);
-				}
-
-			}
-			@files = @new_files;
-		}
-
-		# Read and sort directories
-		rewinddir (DIR);
-		my @directories = grep { !/^\./ && -d "$filename/$_" } readdir (DIR);
-		my $dirsortby;
-		if (defined($r->dir_config('GalleryDirSortBy'))) {
-			$dirsortby=$r->dir_config('GalleryDirSortBy');
-		} else {
-			$dirsortby=$r->dir_config('GallerySortBy');
-		}
-		if ($dirsortby && $dirsortby =~ m/^(size|atime|mtime|ctime)$/) {
-			@directories = map(/^\d+ (.*)/, sort map(stat("$filename/$_")->$dirsortby()." $_", @directories));
-		} else {
-			@directories = sort @directories;
-		}
-
-		closedir(DIR);
-
-
-		# Combine directories and files to one listing
-		my @listing;
-		push (@listing, @directories);
-		push (@listing, @files);
-		push (@listing, @downloadable_files);
-		
-		if (@listing) {
-
-			my $filelist;
-
-			my $file_counter = 0;
-			my $start_at = 1;
-			my $max_files = $r->dir_config('GalleryMaxThumbnailsPerPage');
-
-			if (defined($cgi->param('start'))) {
-				$start_at = $cgi->param('start');
-				if ($start_at < 1) {
-					$start_at = 1;
-				}
-			}
-
-			my $browse_links = "";
-			if (defined($max_files)) {
-			
-				for (my $i=1; $i<=scalar(@listing); $i++) {
-
-					my $from = $i;
-
-					my $to = $i+$max_files-1;
-					if ($to > scalar(@listing)) {
-						$to = scalar(@listing);
-					}
-
-					if ($start_at < $from || $start_at > $to) {
-						$browse_links .= "<a href=\"?start=$from\">$from - ".$to."</a> ";
-					}
-					else {
-						$browse_links .= "$from - $to ";
-					}
-
-					$i+=$max_files-1;
-
-				}
-
-			}
-
-			$tpl_vars{BROWSELINKS} = $browse_links;
-
-			DIRLOOP:
-			foreach my $file (@listing) {
-
-				$file_counter++;
-
-				if ($file_counter < $start_at) {
-					next;
-				}
-
-				if (defined($max_files) && $file_counter > $max_files+$start_at-1) {
-					last DIRLOOP;
-				}
-
-				my $thumbfilename = $topdir."/".$file;
-
-				my $fileurl = $uri."/".$file;
-
-				if (-d $thumbfilename) {
-					my $dirtitle = '';
-					if (-e $thumbfilename . ".folder") {
-						$dirtitle = get_filecontent($thumbfilename . ".folder");
-					}
-
-					$dirtitle = $dirtitle ? $dirtitle : $file;
-					$dirtitle =~ s/_/ /g if $r->dir_config('GalleryUnderscoresToSpaces');
-
-					$tpl_vars{FILES} .= fill_in_file($tpl_vars{directory},
-						HASH=> {FILEURL => uri_escape($fileurl, $escape_rule),
-						FILE    => $dirtitle,
-					});
-
-				}
-				elsif (-f $thumbfilename && $thumbfilename =~ /$doc_pattern/i) {
-					my $type = lc($1);
-					my $stat = stat($thumbfilename);
-					my $size = $stat->size;
-					my $filetype;
-
-					if ($thumbfilename =~ m/\.(mpe?g|avi|mov|asf|wmv)$/i) {
-						$filetype = "video-$type";
-					} elsif ($thumbfilename =~ m/\.(txt|html?)$/i) {
-						$filetype = "text-$type";
-					} elsif ($thumbfilename =~ m/\.(mp3|ogg|wav)$/i) {
-						$filetype = "sound-$type";
-					} elsif ($thumbfilename =~ m/\.(doc|pdf|rtf|csv|eps)$/i) {
-						$filetype = "application-$type";
-					} else {
-						$filetype = "unknown";
-					}
-
-					$tpl_vars{FILES} .= fill_in_file($tpl_vars{file},
-						HASH => {%tpl_vars,
-						FILEURL => uri_escape($fileurl, $escape_rule),
-						ALT => "Size: $size Bytes",
-						FILE => $file,
-						TYPE => $type,
-						FILETYPE => $filetype,
-					});
-
-				}
-				elsif (-f $thumbfilename) {
-
-					my ($width, $height, $type) = imgsize($thumbfilename);
-					next if $type eq 'Data stream is not a known image file format';
-
-					my @filetypes = qw(JPG TIF PNG PPM GIF);
-
-					next unless (grep $type eq $_, @filetypes);
-					my ($thumbnailwidth, $thumbnailheight) = get_thumbnailsize($r, $width, $height);	
-					my $imageinfo = get_imageinfo($r, $thumbfilename, $type, $width, $height);
-					my $cached = get_scaled_picture_name($thumbfilename, $thumbnailwidth, $thumbnailheight);
-
-					my $rotate = readfile_getnum($r, $imageinfo, $thumbfilename.".rotate");
-					my %file_vars = (FILEURL => uri_escape($fileurl, $escape_rule),
-							 FILE    => $file,
-							 DATE    => $imageinfo->{DateTimeOriginal} ? $imageinfo->{DateTimeOriginal} : '', # should this really be a stat of the file instead of ''?
-							 SRC     => uri_escape($uri."/.cache/$cached", $escape_rule),
-							 HEIGHT => (grep($rotate==$_, (1, 3)) ? $thumbnailwidth : $thumbnailheight),
-							 WIDTH => (grep($rotate==$_, (1, 3)) ? $thumbnailheight : $thumbnailwidth),
-							 SELECT  => $select_mode?'<input type="checkbox" name="selection" value="'.$file.'">&nbsp;&nbsp;':'',);
-					$tpl_vars{FILES} .= fill_in_file($tpl_vars{picture},
-									 HASH => {%tpl_vars,
-										  %file_vars});
-				}
-			}
-		}
-		else {
-			$tpl_vars{FILES} = "No files found";
-			$tpl_vars{BROWSELINKS} = "";
-		}
-
-		if (-f $topdir . '.comment') {
-			my $comment_ref = get_comment($topdir . '.comment');
-			my %comment_vars;
-			$comment_vars{COMMENT} = $comment_ref->{COMMENT} . '<br>' if $comment_ref->{COMMENT};
-			$comment_vars{TITLE} = $comment_ref->{TITLE} if $comment_ref->{TITLE};
-			$tpl_vars{DIRCOMMENT} = fill_in_file($tpl_vars{comment},
-							     HASH => \%comment_vars,
-							    );
-			$tpl_vars{TITLE} = $comment_ref->{TITLE} if $comment_ref->{TITLE};
-		} else {
-			$tpl_vars{DIRCOMMENT} = fill_in_file($tpl_vars{nocomment});
-		}
-
-		$tpl_vars{MAIN} = fill_in_file($tpl_vars{index},
-					HASH => \%tpl_vars,
-		);
-		
-		$tpl_vars{MAIN} = fill_in_file($tpl_vars{layout},
-					HASH => \%tpl_vars,
-		);
-
-
+                    }
+                    $index = join '', <CACHE>;
+                    close CACHE;
+                } else {
+                    $index = generate_directory($r, $uri, $cgi, $select_mode, $img_pattern, $doc_pattern, $topdir, $filename);
+                }
 		$r->content_type('text/html');
-		$r->headers_out->{'Content-Length'} = length($tpl_vars{MAIN});
+		$r->headers_out->{'Content-Length'} = length($index);
 
 		if (!$::MP2) {
 			$r->send_http_header;
 		}
 
-		$r->print($tpl_vars{MAIN});
+		$r->print($index);
 		return $::MP2 ? Apache::OK() : Apache::Constants::OK();
 
 	}
@@ -787,6 +586,255 @@
 
 }
 
+#
+# generate_directory: the code to actually generate all the text to
+# return to the user for a directory listing
+#
+sub generate_directory {
+    
+    my ($r, $uri, $cgi, $select_mode, $img_pattern, $doc_pattern, $topdir, $directory) = @_;
+
+    $r->warn("starting to generate_directory $directory");
+
+    my $tpl_dir = $r->dir_config('GalleryTemplateDir');
+
+    my %tpl_vars = (layout    => "$tpl_dir/layout.tpl",
+                    index     => "$tpl_dir/index.tpl",
+                    directory => "$tpl_dir/directory.tpl",
+                    picture   => "$tpl_dir/picture.tpl",
+                    file      => "$tpl_dir/file.tpl",
+                    comment   => "$tpl_dir/dircomment.tpl",
+                    nocomment => "$tpl_dir/nodircomment.tpl",
+                   );
+
+    $tpl_vars{TITLE} = "Index of: $uri";
+    $tpl_vars{META} = " ";
+
+    unless (opendir (DIR, $directory)) {
+        show_error ($r, 500, $!, "Unable to access directory $directory: $!");
+        return $::MP2 ? Apache::OK() : Apache::Constants::OK();
+    }
+
+    $tpl_vars{MENU} = generate_menu($r);
+
+    $tpl_vars{FORM_BEGIN} = $select_mode?'<form method="post">':'';
+    $tpl_vars{FORM_END}   = $select_mode?'</form>':'';
+
+    # Read, sort, and filter files
+    my @files = grep { !/^\./ && -f "$directory/$_" } readdir (DIR);
+
+    @files=gallerysort($r, @files);
+
+    my @downloadable_files;
+
+    if (@files) {
+        # Remove unwanted files from list
+        my @new_files = ();
+        foreach my $picture (@files) {
+
+            my $file = $topdir."/".$picture;
+
+            if ($file =~ /$doc_pattern/i) {
+                push (@downloadable_files, $picture);
+                
+            }
+
+            if ($file =~ /$img_pattern/i) {
+                push (@new_files, $picture);
+            }
+
+        }
+        @files = @new_files;
+    }
+
+    # Read and sort directories
+    rewinddir (DIR);
+    my @directories = grep { !/^\./ && -d "$directory/$_" } readdir (DIR);
+    my $dirsortby;
+    if (defined($r->dir_config('GalleryDirSortBy'))) {
+        $dirsortby=$r->dir_config('GalleryDirSortBy');
+    } else {
+        $dirsortby=$r->dir_config('GallerySortBy');
+    }
+    if ($dirsortby && $dirsortby =~ m/^(size|atime|mtime|ctime)$/) {
+        @directories = map(/^\d+ (.*)/, sort map(stat("$directory/$_")->$dirsortby()." $_", @directories));
+    } else {
+        @directories = sort @directories;
+    }
+
+    closedir(DIR);
+
+
+    # Combine directories and files to one listing
+    my @listing;
+    push (@listing, @directories);
+    push (@listing, @files);
+    push (@listing, @downloadable_files);
+    
+    if (@listing) {
+
+        my $filelist;
+
+        my $file_counter = 0;
+        my $start_at = 1;
+        my $max_files = $r->dir_config('GalleryMaxThumbnailsPerPage');
+
+        if (defined($cgi->param('start'))) {
+            $start_at = $cgi->param('start');
+            if ($start_at < 1) {
+                $start_at = 1;
+            }
+        }
+
+        my $browse_links = "";
+        if (defined($max_files)) {
+            
+            for (my $i=1; $i<=scalar(@listing); $i++) {
+
+                my $from = $i;
+
+                my $to = $i+$max_files-1;
+                if ($to > scalar(@listing)) {
+                    $to = scalar(@listing);
+                }
+
+                if ($start_at < $from || $start_at > $to) {
+                    $browse_links .= "<a href=\"?start=$from\">$from - ".$to."</a> ";
+                }
+                else {
+                    $browse_links .= "$from - $to ";
+                }
+
+                $i+=$max_files-1;
+
+            }
+
+        }
+
+        $tpl_vars{BROWSELINKS} = $browse_links;
+
+      DIRLOOP:
+        foreach my $file (@listing) {
+
+            $file_counter++;
+
+            if ($file_counter < $start_at) {
+                next;
+            }
+
+            if (defined($max_files) && $file_counter > $max_files+$start_at-1) {
+                last DIRLOOP;
+            }
+
+            my $thumbfilename = $topdir."/".$file;
+
+            my $fileurl = $uri."/".$file;
+
+            if (-d $thumbfilename) {
+                my $dirtitle = '';
+                if (-e $thumbfilename . ".folder") {
+                    $dirtitle = get_filecontent($thumbfilename . ".folder");
+                }
+
+                $dirtitle = $dirtitle ? $dirtitle : $file;
+                $dirtitle =~ s/_/ /g if $r->dir_config('GalleryUnderscoresToSpaces');
+
+                $tpl_vars{FILES} .= fill_in_file($tpl_vars{directory},
+                                                 HASH=> {FILEURL => uri_escape($fileurl, $escape_rule),
+                                                         FILE    => $dirtitle,
+                                                        });
+
+            }
+            elsif (-f $thumbfilename && $thumbfilename =~ /$doc_pattern/i) {
+                my $type = lc($1);
+                my $stat = stat($thumbfilename);
+                my $size = $stat->size;
+                my $filetype;
+
+                if ($thumbfilename =~ m/\.(mpe?g|avi|mov|asf|wmv)$/i) {
+                    $filetype = "video-$type";
+                } elsif ($thumbfilename =~ m/\.(txt|html?)$/i) {
+                    $filetype = "text-$type";
+                } elsif ($thumbfilename =~ m/\.(mp3|ogg|wav)$/i) {
+                    $filetype = "sound-$type";
+                } elsif ($thumbfilename =~ m/\.(doc|pdf|rtf|csv|eps)$/i) {
+                    $filetype = "application-$type";
+                } else {
+                    $filetype = "unknown";
+                }
+
+                $tpl_vars{FILES} .= fill_in_file($tpl_vars{file},
+                                                 HASH => {%tpl_vars,
+                                                          FILEURL => uri_escape($fileurl, $escape_rule),
+                                                          ALT => "Size: $size Bytes",
+                                                          FILE => $file,
+                                                          TYPE => $type,
+                                                          FILETYPE => $filetype,
+                                                         });
+
+            }
+            elsif (-f $thumbfilename) {
+
+                my ($width, $height, $type) = imgsize($thumbfilename);
+                next if $type eq 'Data stream is not a known image file format';
+
+                my @filetypes = qw(JPG TIF PNG PPM GIF);
+
+                next unless (grep $type eq $_, @filetypes);
+                my ($thumbnailwidth, $thumbnailheight) = get_thumbnailsize($r, $width, $height);	
+                my $imageinfo = get_imageinfo($r, $thumbfilename, $type, $width, $height);
+                my $cached = get_scaled_picture_name($thumbfilename, $thumbnailwidth, $thumbnailheight);
+
+                my $rotate = readfile_getnum($r, $imageinfo, $thumbfilename.".rotate");
+                my %file_vars = (FILEURL => uri_escape($fileurl, $escape_rule),
+                                 FILE    => $file,
+                                 DATE    => $imageinfo->{DateTimeOriginal} ? $imageinfo->{DateTimeOriginal} : '', # should this really be a stat of the file instead of ''?
+                                 SRC     => uri_escape($uri."/.cache/$cached", $escape_rule),
+                                 HEIGHT => (grep($rotate==$_, (1, 3)) ? $thumbnailwidth : $thumbnailheight),
+                                 WIDTH => (grep($rotate==$_, (1, 3)) ? $thumbnailheight : $thumbnailwidth),
+                                 SELECT  => $select_mode?'<input type="checkbox" name="selection" value="'.$file.'">&nbsp;&nbsp;':'',);
+                $tpl_vars{FILES} .= fill_in_file($tpl_vars{picture},
+                                                 HASH => {%tpl_vars,
+                                                          %file_vars});
+            }
+        }
+    }
+    else {
+        $tpl_vars{FILES} = "No files found";
+        $tpl_vars{BROWSELINKS} = "";
+    }
+
+    if (-f $topdir . '.comment') {
+        my $comment_ref = get_comment($topdir . '.comment');
+        my %comment_vars;
+        $comment_vars{COMMENT} = $comment_ref->{COMMENT} . '<br>' if $comment_ref->{COMMENT};
+        $comment_vars{TITLE} = $comment_ref->{TITLE} if $comment_ref->{TITLE};
+        $tpl_vars{DIRCOMMENT} = fill_in_file($tpl_vars{comment},
+                                             HASH => \%comment_vars,
+                                            );
+        $tpl_vars{TITLE} = $comment_ref->{TITLE} if $comment_ref->{TITLE};
+    } else {
+        $tpl_vars{DIRCOMMENT} = fill_in_file($tpl_vars{nocomment});
+    }
+
+    $tpl_vars{MAIN} = fill_in_file($tpl_vars{index},
+                                   HASH => \%tpl_vars,
+                                  );
+    
+    $tpl_vars{MAIN} = fill_in_file($tpl_vars{layout},
+                                   HASH => \%tpl_vars,
+                                  );
+
+    my $cachefile = cache_dir($r, 0) . "/index.html";
+    open CACHE, ">$cachefile" or
+        $r->warn("Can't write '$cachefile', $!\n");
+    print CACHE $tpl_vars{MAIN};
+    close CACHE;
+
+    $r->warn("finished generating_directory $directory");
+    return $tpl_vars{MAIN};
+}
+
 sub cache_dir {
 
 	my ($r, $strip_filename) = @_;
-- 
[email protected]