Mail Index
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: [ApacheGallery] directory generating slowness
On Thu, Dec 09, 2004 at 08:34:13AM -0500, Doug Alcorn wrote:
> I'm bothered by the amount of time that A::G takes to generate
> directory listings. I've looked briefly at that code. It seems like
> it's doing a lot of fast things in sequence that ends up being overall
> slow.
Here's my first patch. It yanks all the stuff out of the handler for
generating the directory page and makes a sub generate_directory. The
actual page generation code is unchanged except that
generate_directory dumps it's output to cache_dir($r, 0) .
"/index.html". The the handler sub checks the timestamp on the
.cache/index.html versus the timestamp of the image directory and
either serves up the cached index or generates a new one. I made a
half-hearted attempt to determine when A::G was instantiated, but was
unable to make it work like it should.
You can use this patch or /dev/null it.
--- Apache-Gallery-0.9.1/lib/Apache/Gallery.pm Sat Sep 11 17:53:05 2004
+++ Apache-Gallery-devel/lib/Apache/Gallery.pm Thu Dec 9 11:56:58 2004
@@ -5,9 +5,10 @@
use strict;
-use vars qw($VERSION);
+use vars qw($VERSION $GALLERY_STARTTIME);
$VERSION = "0.9.1";
+$GALLERY_STARTTIME = "";
BEGIN {
@@ -39,10 +40,10 @@
Apache::Constants->import('OK','DECLINED','FORBIDDEN','NOT_FOUND');
}
-
+ $::GALLERY_STARTTIME = time;
}
use Image::Info qw(image_info);
use Image::Size qw(imgsize);
use Image::Imlib2;
use Text::Template qw(fill_in_file);
@@ -182,244 +192,30 @@
}
}
- my $tpl_dir = $r->dir_config('GalleryTemplateDir');
-
- my %tpl_vars = (layout => "$tpl_dir/layout.tpl",
- index => "$tpl_dir/index.tpl",
- directory => "$tpl_dir/directory.tpl",
- picture => "$tpl_dir/picture.tpl",
- file => "$tpl_dir/file.tpl",
- comment => "$tpl_dir/dircomment.tpl",
- nocomment => "$tpl_dir/nodircomment.tpl",
- );
-
- $tpl_vars{TITLE} = "Index of: $uri";
- $tpl_vars{META} = " ";
-
- unless (opendir (DIR, $filename)) {
- show_error ($r, 500, $!, "Unable to access directory $filename: $!");
+ my $cacheindex = cache_dir($r, 0) . "/index.html";
+ my $index;
+ my $save_index = 0;
+ if ( -f $cacheindex && -r _
+ && stat($cacheindex)->mtime() > stat(cache_dir($r, 0))->mtime() ) {
+# && stat($cacheindex)->mtime() > $::GALLERY_STARTTIME) {
+ unless (open CACHE, $cacheindex) {
+ $r->warn("Can't read '$cacheindex', $!\n");
+ show_error($r, 404, "Unable to access file", "Unable to access directory $cacheindex");
return $::MP2 ? Apache::OK() : Apache::Constants::OK();
- }
-
- $tpl_vars{MENU} = generate_menu($r);
-
- $tpl_vars{FORM_BEGIN} = $select_mode?'<form method="post">':'';
- $tpl_vars{FORM_END} = $select_mode?'</form>':'';
-
- # Read, sort, and filter files
- my @files = grep { !/^\./ && -f "$filename/$_" } readdir (DIR);
-
- @files=gallerysort($r, @files);
-
- my @downloadable_files;
-
- if (@files) {
- # Remove unwanted files from list
- my @new_files = ();
- foreach my $picture (@files) {
-
- my $file = $topdir."/".$picture;
-
- if ($file =~ /$doc_pattern/i) {
- push (@downloadable_files, $picture);
-
- }
-
- if ($file =~ /$img_pattern/i) {
- push (@new_files, $picture);
- }
-
- }
- @files = @new_files;
- }
-
- # Read and sort directories
- rewinddir (DIR);
- my @directories = grep { !/^\./ && -d "$filename/$_" } readdir (DIR);
- my $dirsortby;
- if (defined($r->dir_config('GalleryDirSortBy'))) {
- $dirsortby=$r->dir_config('GalleryDirSortBy');
- } else {
- $dirsortby=$r->dir_config('GallerySortBy');
- }
- if ($dirsortby && $dirsortby =~ m/^(size|atime|mtime|ctime)$/) {
- @directories = map(/^\d+ (.*)/, sort map(stat("$filename/$_")->$dirsortby()." $_", @directories));
- } else {
- @directories = sort @directories;
- }
-
- closedir(DIR);
-
-
- # Combine directories and files to one listing
- my @listing;
- push (@listing, @directories);
- push (@listing, @files);
- push (@listing, @downloadable_files);
-
- if (@listing) {
-
- my $filelist;
-
- my $file_counter = 0;
- my $start_at = 1;
- my $max_files = $r->dir_config('GalleryMaxThumbnailsPerPage');
-
- if (defined($cgi->param('start'))) {
- $start_at = $cgi->param('start');
- if ($start_at < 1) {
- $start_at = 1;
- }
- }
-
- my $browse_links = "";
- if (defined($max_files)) {
-
- for (my $i=1; $i<=scalar(@listing); $i++) {
-
- my $from = $i;
-
- my $to = $i+$max_files-1;
- if ($to > scalar(@listing)) {
- $to = scalar(@listing);
- }
-
- if ($start_at < $from || $start_at > $to) {
- $browse_links .= "<a href=\"?start=$from\">$from - ".$to."</a> ";
- }
- else {
- $browse_links .= "$from - $to ";
- }
-
- $i+=$max_files-1;
-
- }
-
- }
-
- $tpl_vars{BROWSELINKS} = $browse_links;
-
- DIRLOOP:
- foreach my $file (@listing) {
-
- $file_counter++;
-
- if ($file_counter < $start_at) {
- next;
- }
-
- if (defined($max_files) && $file_counter > $max_files+$start_at-1) {
- last DIRLOOP;
- }
-
- my $thumbfilename = $topdir."/".$file;
-
- my $fileurl = $uri."/".$file;
-
- if (-d $thumbfilename) {
- my $dirtitle = '';
- if (-e $thumbfilename . ".folder") {
- $dirtitle = get_filecontent($thumbfilename . ".folder");
- }
-
- $dirtitle = $dirtitle ? $dirtitle : $file;
- $dirtitle =~ s/_/ /g if $r->dir_config('GalleryUnderscoresToSpaces');
-
- $tpl_vars{FILES} .= fill_in_file($tpl_vars{directory},
- HASH=> {FILEURL => uri_escape($fileurl, $escape_rule),
- FILE => $dirtitle,
- });
-
- }
- elsif (-f $thumbfilename && $thumbfilename =~ /$doc_pattern/i) {
- my $type = lc($1);
- my $stat = stat($thumbfilename);
- my $size = $stat->size;
- my $filetype;
-
- if ($thumbfilename =~ m/\.(mpe?g|avi|mov|asf|wmv)$/i) {
- $filetype = "video-$type";
- } elsif ($thumbfilename =~ m/\.(txt|html?)$/i) {
- $filetype = "text-$type";
- } elsif ($thumbfilename =~ m/\.(mp3|ogg|wav)$/i) {
- $filetype = "sound-$type";
- } elsif ($thumbfilename =~ m/\.(doc|pdf|rtf|csv|eps)$/i) {
- $filetype = "application-$type";
- } else {
- $filetype = "unknown";
- }
-
- $tpl_vars{FILES} .= fill_in_file($tpl_vars{file},
- HASH => {%tpl_vars,
- FILEURL => uri_escape($fileurl, $escape_rule),
- ALT => "Size: $size Bytes",
- FILE => $file,
- TYPE => $type,
- FILETYPE => $filetype,
- });
-
- }
- elsif (-f $thumbfilename) {
-
- my ($width, $height, $type) = imgsize($thumbfilename);
- next if $type eq 'Data stream is not a known image file format';
-
- my @filetypes = qw(JPG TIF PNG PPM GIF);
-
- next unless (grep $type eq $_, @filetypes);
- my ($thumbnailwidth, $thumbnailheight) = get_thumbnailsize($r, $width, $height);
- my $imageinfo = get_imageinfo($r, $thumbfilename, $type, $width, $height);
- my $cached = get_scaled_picture_name($thumbfilename, $thumbnailwidth, $thumbnailheight);
-
- my $rotate = readfile_getnum($r, $imageinfo, $thumbfilename.".rotate");
- my %file_vars = (FILEURL => uri_escape($fileurl, $escape_rule),
- FILE => $file,
- DATE => $imageinfo->{DateTimeOriginal} ? $imageinfo->{DateTimeOriginal} : '', # should this really be a stat of the file instead of ''?
- SRC => uri_escape($uri."/.cache/$cached", $escape_rule),
- HEIGHT => (grep($rotate==$_, (1, 3)) ? $thumbnailwidth : $thumbnailheight),
- WIDTH => (grep($rotate==$_, (1, 3)) ? $thumbnailheight : $thumbnailwidth),
- SELECT => $select_mode?'<input type="checkbox" name="selection" value="'.$file.'"> ':'',);
- $tpl_vars{FILES} .= fill_in_file($tpl_vars{picture},
- HASH => {%tpl_vars,
- %file_vars});
- }
- }
- }
- else {
- $tpl_vars{FILES} = "No files found";
- $tpl_vars{BROWSELINKS} = "";
- }
-
- if (-f $topdir . '.comment') {
- my $comment_ref = get_comment($topdir . '.comment');
- my %comment_vars;
- $comment_vars{COMMENT} = $comment_ref->{COMMENT} . '<br>' if $comment_ref->{COMMENT};
- $comment_vars{TITLE} = $comment_ref->{TITLE} if $comment_ref->{TITLE};
- $tpl_vars{DIRCOMMENT} = fill_in_file($tpl_vars{comment},
- HASH => \%comment_vars,
- );
- $tpl_vars{TITLE} = $comment_ref->{TITLE} if $comment_ref->{TITLE};
- } else {
- $tpl_vars{DIRCOMMENT} = fill_in_file($tpl_vars{nocomment});
- }
-
- $tpl_vars{MAIN} = fill_in_file($tpl_vars{index},
- HASH => \%tpl_vars,
- );
-
- $tpl_vars{MAIN} = fill_in_file($tpl_vars{layout},
- HASH => \%tpl_vars,
- );
-
-
+ }
+ $index = join '', <CACHE>;
+ close CACHE;
+ } else {
+ $index = generate_directory($r, $uri, $cgi, $select_mode, $img_pattern, $doc_pattern, $topdir, $filename);
+ }
$r->content_type('text/html');
- $r->headers_out->{'Content-Length'} = length($tpl_vars{MAIN});
+ $r->headers_out->{'Content-Length'} = length($index);
if (!$::MP2) {
$r->send_http_header;
}
- $r->print($tpl_vars{MAIN});
+ $r->print($index);
return $::MP2 ? Apache::OK() : Apache::Constants::OK();
}
@@ -787,6 +586,255 @@
}
+#
+# generate_directory: the code to actually generate all the text to
+# return to the user for a directory listing
+#
+sub generate_directory {
+
+ my ($r, $uri, $cgi, $select_mode, $img_pattern, $doc_pattern, $topdir, $directory) = @_;
+
+ $r->warn("starting to generate_directory $directory");
+
+ my $tpl_dir = $r->dir_config('GalleryTemplateDir');
+
+ my %tpl_vars = (layout => "$tpl_dir/layout.tpl",
+ index => "$tpl_dir/index.tpl",
+ directory => "$tpl_dir/directory.tpl",
+ picture => "$tpl_dir/picture.tpl",
+ file => "$tpl_dir/file.tpl",
+ comment => "$tpl_dir/dircomment.tpl",
+ nocomment => "$tpl_dir/nodircomment.tpl",
+ );
+
+ $tpl_vars{TITLE} = "Index of: $uri";
+ $tpl_vars{META} = " ";
+
+ unless (opendir (DIR, $directory)) {
+ show_error ($r, 500, $!, "Unable to access directory $directory: $!");
+ return $::MP2 ? Apache::OK() : Apache::Constants::OK();
+ }
+
+ $tpl_vars{MENU} = generate_menu($r);
+
+ $tpl_vars{FORM_BEGIN} = $select_mode?'<form method="post">':'';
+ $tpl_vars{FORM_END} = $select_mode?'</form>':'';
+
+ # Read, sort, and filter files
+ my @files = grep { !/^\./ && -f "$directory/$_" } readdir (DIR);
+
+ @files=gallerysort($r, @files);
+
+ my @downloadable_files;
+
+ if (@files) {
+ # Remove unwanted files from list
+ my @new_files = ();
+ foreach my $picture (@files) {
+
+ my $file = $topdir."/".$picture;
+
+ if ($file =~ /$doc_pattern/i) {
+ push (@downloadable_files, $picture);
+
+ }
+
+ if ($file =~ /$img_pattern/i) {
+ push (@new_files, $picture);
+ }
+
+ }
+ @files = @new_files;
+ }
+
+ # Read and sort directories
+ rewinddir (DIR);
+ my @directories = grep { !/^\./ && -d "$directory/$_" } readdir (DIR);
+ my $dirsortby;
+ if (defined($r->dir_config('GalleryDirSortBy'))) {
+ $dirsortby=$r->dir_config('GalleryDirSortBy');
+ } else {
+ $dirsortby=$r->dir_config('GallerySortBy');
+ }
+ if ($dirsortby && $dirsortby =~ m/^(size|atime|mtime|ctime)$/) {
+ @directories = map(/^\d+ (.*)/, sort map(stat("$directory/$_")->$dirsortby()." $_", @directories));
+ } else {
+ @directories = sort @directories;
+ }
+
+ closedir(DIR);
+
+
+ # Combine directories and files to one listing
+ my @listing;
+ push (@listing, @directories);
+ push (@listing, @files);
+ push (@listing, @downloadable_files);
+
+ if (@listing) {
+
+ my $filelist;
+
+ my $file_counter = 0;
+ my $start_at = 1;
+ my $max_files = $r->dir_config('GalleryMaxThumbnailsPerPage');
+
+ if (defined($cgi->param('start'))) {
+ $start_at = $cgi->param('start');
+ if ($start_at < 1) {
+ $start_at = 1;
+ }
+ }
+
+ my $browse_links = "";
+ if (defined($max_files)) {
+
+ for (my $i=1; $i<=scalar(@listing); $i++) {
+
+ my $from = $i;
+
+ my $to = $i+$max_files-1;
+ if ($to > scalar(@listing)) {
+ $to = scalar(@listing);
+ }
+
+ if ($start_at < $from || $start_at > $to) {
+ $browse_links .= "<a href=\"?start=$from\">$from - ".$to."</a> ";
+ }
+ else {
+ $browse_links .= "$from - $to ";
+ }
+
+ $i+=$max_files-1;
+
+ }
+
+ }
+
+ $tpl_vars{BROWSELINKS} = $browse_links;
+
+ DIRLOOP:
+ foreach my $file (@listing) {
+
+ $file_counter++;
+
+ if ($file_counter < $start_at) {
+ next;
+ }
+
+ if (defined($max_files) && $file_counter > $max_files+$start_at-1) {
+ last DIRLOOP;
+ }
+
+ my $thumbfilename = $topdir."/".$file;
+
+ my $fileurl = $uri."/".$file;
+
+ if (-d $thumbfilename) {
+ my $dirtitle = '';
+ if (-e $thumbfilename . ".folder") {
+ $dirtitle = get_filecontent($thumbfilename . ".folder");
+ }
+
+ $dirtitle = $dirtitle ? $dirtitle : $file;
+ $dirtitle =~ s/_/ /g if $r->dir_config('GalleryUnderscoresToSpaces');
+
+ $tpl_vars{FILES} .= fill_in_file($tpl_vars{directory},
+ HASH=> {FILEURL => uri_escape($fileurl, $escape_rule),
+ FILE => $dirtitle,
+ });
+
+ }
+ elsif (-f $thumbfilename && $thumbfilename =~ /$doc_pattern/i) {
+ my $type = lc($1);
+ my $stat = stat($thumbfilename);
+ my $size = $stat->size;
+ my $filetype;
+
+ if ($thumbfilename =~ m/\.(mpe?g|avi|mov|asf|wmv)$/i) {
+ $filetype = "video-$type";
+ } elsif ($thumbfilename =~ m/\.(txt|html?)$/i) {
+ $filetype = "text-$type";
+ } elsif ($thumbfilename =~ m/\.(mp3|ogg|wav)$/i) {
+ $filetype = "sound-$type";
+ } elsif ($thumbfilename =~ m/\.(doc|pdf|rtf|csv|eps)$/i) {
+ $filetype = "application-$type";
+ } else {
+ $filetype = "unknown";
+ }
+
+ $tpl_vars{FILES} .= fill_in_file($tpl_vars{file},
+ HASH => {%tpl_vars,
+ FILEURL => uri_escape($fileurl, $escape_rule),
+ ALT => "Size: $size Bytes",
+ FILE => $file,
+ TYPE => $type,
+ FILETYPE => $filetype,
+ });
+
+ }
+ elsif (-f $thumbfilename) {
+
+ my ($width, $height, $type) = imgsize($thumbfilename);
+ next if $type eq 'Data stream is not a known image file format';
+
+ my @filetypes = qw(JPG TIF PNG PPM GIF);
+
+ next unless (grep $type eq $_, @filetypes);
+ my ($thumbnailwidth, $thumbnailheight) = get_thumbnailsize($r, $width, $height);
+ my $imageinfo = get_imageinfo($r, $thumbfilename, $type, $width, $height);
+ my $cached = get_scaled_picture_name($thumbfilename, $thumbnailwidth, $thumbnailheight);
+
+ my $rotate = readfile_getnum($r, $imageinfo, $thumbfilename.".rotate");
+ my %file_vars = (FILEURL => uri_escape($fileurl, $escape_rule),
+ FILE => $file,
+ DATE => $imageinfo->{DateTimeOriginal} ? $imageinfo->{DateTimeOriginal} : '', # should this really be a stat of the file instead of ''?
+ SRC => uri_escape($uri."/.cache/$cached", $escape_rule),
+ HEIGHT => (grep($rotate==$_, (1, 3)) ? $thumbnailwidth : $thumbnailheight),
+ WIDTH => (grep($rotate==$_, (1, 3)) ? $thumbnailheight : $thumbnailwidth),
+ SELECT => $select_mode?'<input type="checkbox" name="selection" value="'.$file.'"> ':'',);
+ $tpl_vars{FILES} .= fill_in_file($tpl_vars{picture},
+ HASH => {%tpl_vars,
+ %file_vars});
+ }
+ }
+ }
+ else {
+ $tpl_vars{FILES} = "No files found";
+ $tpl_vars{BROWSELINKS} = "";
+ }
+
+ if (-f $topdir . '.comment') {
+ my $comment_ref = get_comment($topdir . '.comment');
+ my %comment_vars;
+ $comment_vars{COMMENT} = $comment_ref->{COMMENT} . '<br>' if $comment_ref->{COMMENT};
+ $comment_vars{TITLE} = $comment_ref->{TITLE} if $comment_ref->{TITLE};
+ $tpl_vars{DIRCOMMENT} = fill_in_file($tpl_vars{comment},
+ HASH => \%comment_vars,
+ );
+ $tpl_vars{TITLE} = $comment_ref->{TITLE} if $comment_ref->{TITLE};
+ } else {
+ $tpl_vars{DIRCOMMENT} = fill_in_file($tpl_vars{nocomment});
+ }
+
+ $tpl_vars{MAIN} = fill_in_file($tpl_vars{index},
+ HASH => \%tpl_vars,
+ );
+
+ $tpl_vars{MAIN} = fill_in_file($tpl_vars{layout},
+ HASH => \%tpl_vars,
+ );
+
+ my $cachefile = cache_dir($r, 0) . "/index.html";
+ open CACHE, ">$cachefile" or
+ $r->warn("Can't write '$cachefile', $!\n");
+ print CACHE $tpl_vars{MAIN};
+ close CACHE;
+
+ $r->warn("finished generating_directory $directory");
+ return $tpl_vars{MAIN};
+}
+
sub cache_dir {
my ($r, $strip_filename) = @_;
--
doug@xxxxxxxxx