[Jifty-commit] jifty branch, master, updated. 9196d3d0661490d6064ec8044f1e06ad8a72b950

Jifty commits jifty-commit at lists.jifty.org
Wed Jan 27 17:34:19 EST 2010


The branch, master has been updated
       via  9196d3d0661490d6064ec8044f1e06ad8a72b950 (commit)
       via  5396e2e35b96b0aebc9f3086b0b90e12f57aeea8 (commit)
       via  44359d5a34b95bcd2acf238f6d3eff815c235f03 (commit)
       via  409707d0b5faa01b126cb26aad4edea883a7c975 (commit)
       via  57a4fcc33ad624d6004913cf80e0ee69775e5755 (commit)
       via  75d11b59bfb3cbb381ec76d6b6de606aac9a3933 (commit)
       via  d572a6d5e94bf7f2389ef035d460a174e8395978 (commit)
      from  8dfd20ed1bb64a0b383f821082f6c4864270c98f (commit)

Summary of changes:
 Makefile.PL                        |    5 +
 lib/Jifty.pm                       |    6 +
 lib/Jifty/CAS.pm                   |   50 +++--------
 lib/Jifty/CAS/Blob.pm              |   14 ++-
 lib/Jifty/{CAS.pm => CAS/Store.pm} |   47 ++++------
 lib/Jifty/CAS/Store/Memcached.pm   |  178 ++++++++++++++++++++++++++++++++++++
 lib/Jifty/Config.pm                |   10 ++
 7 files changed, 242 insertions(+), 68 deletions(-)
 copy lib/Jifty/{CAS.pm => CAS/Store.pm} (53%)
 create mode 100644 lib/Jifty/CAS/Store/Memcached.pm

- Log -----------------------------------------------------------------
commit d572a6d5e94bf7f2389ef035d460a174e8395978
Author: Thomas Sibley <trs at bestpractical.com>
Date:   Tue Jan 26 17:23:54 2010 -0500

    Only deflate CAS content upon first request for it

diff --git a/lib/Jifty/CAS/Blob.pm b/lib/Jifty/CAS/Blob.pm
index 1915633..2f7ffaf 100644
--- a/lib/Jifty/CAS/Blob.pm
+++ b/lib/Jifty/CAS/Blob.pm
@@ -60,7 +60,7 @@ Retuens the key calculated for this content.
 
 =cut
 
-__PACKAGE__->mk_accessors(qw(content content_deflated metadata key));
+__PACKAGE__->mk_accessors(qw(content metadata key));
 
 sub new {
     my $class = shift;
@@ -71,9 +71,17 @@ sub new {
         %$args,
     } );
     $self->key( md5_hex( $self->metadata->{hash_with} || $self->content ) );
-    $self->content_deflated( Compress::Zlib::memGzip( $self->content ) )
-        if $self->metadata->{deflate};
     return $self;
 }
 
+sub content_deflated {
+    my $self = shift;
+    return unless $self->metadata->{deflate};
+
+    $self->{content_deflated} = Compress::Zlib::memGzip( $self->content )
+        unless exists $self->{content_deflated};
+
+    return $self->{content_deflated};
+}
+
 1;

commit 75d11b59bfb3cbb381ec76d6b6de606aac9a3933
Author: Thomas Sibley <trs at bestpractical.com>
Date:   Tue Jan 26 20:54:35 2010 -0500

    Jifty::CAS now has a memcached backend

diff --git a/lib/Jifty.pm b/lib/Jifty.pm
index 10c1907..b0dd454 100644
--- a/lib/Jifty.pm
+++ b/lib/Jifty.pm
@@ -166,6 +166,12 @@ sub new {
     Jifty::Util->require( $record_base_class );
     push @Jifty::Record::ISA, $record_base_class unless $record_base_class eq 'Jifty::Record';
 
+    # Configure the base class for Jifty::CAS
+    @Jifty::CAS::ISA = grep { $_ ne 'Jifty::CAS::Store' } @Jifty::CAS::ISA;
+    my $cas_base = Jifty->config->framework('CAS')->{'BaseClass'};
+    Jifty::Util->require( $cas_base );
+    push @Jifty::CAS::ISA, $cas_base unless $cas_base eq 'Jifty::CAS';
+
     # Logger turn on
     Jifty->logger( Jifty::Logger->new( $args{'logger_component'} ) );
 
diff --git a/lib/Jifty/CAS.pm b/lib/Jifty/CAS.pm
index 1e4829e..63eaaeb 100644
--- a/lib/Jifty/CAS.pm
+++ b/lib/Jifty/CAS.pm
@@ -1,9 +1,12 @@
-package Jifty::CAS;
 use strict;
+use warnings;
+
+package Jifty::CAS;
+use base 'Jifty::CAS::Store';
 
 =head1 NAME
 
-Jifty::CAS - Jifty's Content-addressable storage facility
+Jifty::CAS - Jifty's Content-Addressable Storage facility
 
 =head1 SYNOPSIS
 
@@ -28,11 +31,13 @@ stored under a "domain", and can be addressed using wither the "key",
 which is an C<md5> sum, or the "name", which simply stores the most
 recent key provided with that name.
 
-=cut
+=head1 BACKENDS
 
-my %CONTAINER;
-
-use Jifty::CAS::Blob;
+The default data store is an per-process, in-memory store.  A
+L<memcached|Jifty::CAS::Store::Memcached> backed store is also available and
+has the benefits of sharing the cache across all instances of a Jifty app using
+Jifty::CAS.  The memcached store is limited to objects less than 1MB in size,
+however.
 
 =head1 METHODS
 
@@ -42,37 +47,11 @@ Publishes the given C<CONTENT> at the address C<DOMAIN> and C<NAME>.
 C<METADATA> is an arbitrary hash; see L<Jifty::CAS::Blob> for more.
 Returns the key.
 
-=cut
-
-sub publish {
-    my ($class, $domain, $name, $content, $opt) = @_;
-    my $db = $CONTAINER{$domain} ||= {};
-    $opt ||= {};
-
-    my $blob = Jifty::CAS::Blob->new(
-        {   content  => $content,
-            metadata => $opt,
-        }
-    );
-    my $key = $blob->key;
-    $db->{DB}{$key} = $blob;
-    $db->{KEYS}{$name} = $key;
-
-    return $key;
-}
-
 =head2 key DOMAIN NAME
 
 Returns the most recent key for the given pair of C<DOMAIN> and
 C<NAME>, or undef if none such exists.
 
-=cut
-
-sub key {
-    my ($class, $domain, $name) = @_;
-    return $CONTAINER{$domain}{KEYS}{$name};
-}
-
 =head2 retrieve DOMAIN KEY
 
 Returns a L<Jifty::CAS::Blob> for the given pair of C<DOMAIN> and
@@ -80,9 +59,4 @@ C<KEY>, or undef if none such exists.
 
 =cut
 
-sub retrieve {
-    my ($class, $domain, $key) = @_;
-    return $CONTAINER{$domain}{DB}{$key};
-}
-
 1;
diff --git a/lib/Jifty/CAS.pm b/lib/Jifty/CAS/Store.pm
similarity index 53%
copy from lib/Jifty/CAS.pm
copy to lib/Jifty/CAS/Store.pm
index 1e4829e..390955d 100644
--- a/lib/Jifty/CAS.pm
+++ b/lib/Jifty/CAS/Store.pm
@@ -1,40 +1,23 @@
-package Jifty::CAS;
 use strict;
+use warnings;
 
-=head1 NAME
-
-Jifty::CAS - Jifty's Content-addressable storage facility
-
-=head1 SYNOPSIS
-
-  my $key = Jifty::CAS->publish('js' => 'all', $content,
-                      { hash_with => $content, # default behaviour
-                        content_type => 'application/x-javascript',
-                        deflate => 1
-                      });
+package Jifty::CAS::Store;
 
-  $ie_key = Jifty::CAS->publish('js' => 'ie-only', $ie_content,
-                      { hash_with => $ie_content,
-                        content_type => 'application/x-javascript',
-                      });
+=head1 NAME
 
-  $key = Jifty::CAS->key('js', 'ie-only');
-  my $blob = Jifty::CAS->retrieve('js', $key);
+Jifty::CAS::Store - The default, per-process, in-memory store for Jifty's
+Content-Addressable Storage facility
 
 =head1 DESCRIPTION
 
-Provides an in-memory C<md5>-addressed content store.  Content is
-stored under a "domain", and can be addressed using wither the "key",
-which is an C<md5> sum, or the "name", which simply stores the most
-recent key provided with that name.
+This is the default backend store for L<Jifty::CAS>.  For more information, see
+L<Jifty::CAS/DESCRIPTION>.
 
 =cut
 
-my %CONTAINER;
-
 use Jifty::CAS::Blob;
 
-=head1 METHODS
+my %CONTAINER;
 
 =head2 publish DOMAIN NAME CONTENT METADATA
 
@@ -46,7 +29,6 @@ Returns the key.
 
 sub publish {
     my ($class, $domain, $name, $content, $opt) = @_;
-    my $db = $CONTAINER{$domain} ||= {};
     $opt ||= {};
 
     my $blob = Jifty::CAS::Blob->new(
@@ -54,10 +36,21 @@ sub publish {
             metadata => $opt,
         }
     );
+    return $class->_store( $domain, $name, $blob );
+}
+
+=head2 _store DOMAIN NAME BLOB
+
+Stores the BLOB (a L<Jifty::CAS::Blob>) in the backend.  Returns the key.  Don't use this directly, use C<publish> instead.
+
+=cut
+
+sub _store {
+    my ($class, $domain, $name, $blob) = @_;
+    my $db  = $CONTAINER{$domain} ||= {};
     my $key = $blob->key;
     $db->{DB}{$key} = $blob;
     $db->{KEYS}{$name} = $key;
-
     return $key;
 }
 
diff --git a/lib/Jifty/CAS/Store/Memcached.pm b/lib/Jifty/CAS/Store/Memcached.pm
new file mode 100644
index 0000000..cb5ba8b
--- /dev/null
+++ b/lib/Jifty/CAS/Store/Memcached.pm
@@ -0,0 +1,120 @@
+use strict;
+use warnings;
+
+package Jifty::CAS::Store::Memcached;
+
+use base 'Jifty::CAS::Store';
+
+=head1 NAME
+
+Jifty::CAS::Store::Memcached - A memcached backend for Jifty's
+Content-Addressable Storage facility
+
+=head1 SYNOPSIS
+
+Add the following to your Jifty config.yml:
+
+    framework:
+      CAS:
+        BaseClass: Jifty::CAS::Store::Memcached
+
+=head1 DESCRIPTION
+
+This is a memcached backend for L<Jifty::CAS>.  For more information, see
+L<Jifty::CAS/DESCRIPTION>.
+
+=cut
+
+use Cache::Memcached;
+
+our $MEMCACHED;
+
+
+=head1 METHODS
+
+=head2 memcached
+
+Returns the L<Cache::Memcached> object for this class.
+
+=cut
+
+sub memcached {
+    $MEMCACHED ||= Cache::Memcached->new( $_[0]->memcached_config );
+}
+
+=head2 _store DOMAIN NAME BLOB
+
+Stores the BLOB (a L<Jifty::CAS::Blob>) in memcached.  Returns the key.
+
+=cut
+
+sub _store {
+    my ($class, $domain, $name, $blob) = @_;
+
+    # my $db  = $CONTAINER{$domain} ||= {};
+    # $db->{DB}{$key} = $blob;
+    # $db->{KEYS}{$name} = $key;
+
+    my $key = $blob->key;
+    $class->memcached->set("$domain:db:$key", $blob);
+    $class->memcached->set("$domain:keys:$name", $key);
+
+    return $key;
+}
+
+=head2 key DOMAIN NAME
+
+Returns the most recent key for the given pair of C<DOMAIN> and
+C<NAME>, or undef if none such exists.
+
+=cut
+
+sub key {
+    my ($class, $domain, $name) = @_;
+    return $class->memcached->get("$domain:keys:$name");
+}
+
+=head2 retrieve DOMAIN KEY
+
+Returns a L<Jifty::CAS::Blob> for the given pair of C<DOMAIN> and
+C<KEY>, or undef if none such exists.
+
+=cut
+
+sub retrieve {
+    my ($class, $domain, $key) = @_;
+    return $class->memcached->get("$domain:db:$key");
+}
+
+=head2 memcached_config
+
+Returns a hashref containing arguments to pass to L<Cache::Memcached> during
+construction. The defaults are like:
+
+  {
+      servers     => [ '127.0.0.1:11211' ],
+      debug       => 0,
+      namespace   => Jifty->config->framework('ApplicationName'),
+      compress_threshold => 10240,
+  }
+
+To change these options, set them in your Jifty application config file under
+C</framework/CAS/Memcached> like so:
+
+    framework:
+      CAS:
+        BaseClass: Jifty::CAS::Store::Memcached
+        Memcached:
+            servers:
+                - 10.0.0.2:11211
+                - 10.0.0.3:11211
+            compress_threshold: 5120
+
+=cut
+
+sub memcached_config {
+    Jifty->config->framework('CAS')->{'Memcached'}
+        || Jifty->config->defaults->{'framework'}{'CAS'}{'Memcached'}
+}
+
+1;
diff --git a/lib/Jifty/Config.pm b/lib/Jifty/Config.pm
index 1e33a11..f2c4433 100644
--- a/lib/Jifty/Config.pm
+++ b/lib/Jifty/Config.pm
@@ -614,6 +614,15 @@ sub defaults {
                 DefaultTemplateRoot => Jifty::Util->share_root . '/web/templates',
                 SessionCookieName => 'JIFTY_SID_$PORT',
             },
+            CAS => {
+                BaseClass => 'Jifty::CAS::Store',
+                Memcached => {
+                    servers     => [ '127.0.0.1:11211' ],
+                    debug       => 0,
+                    namespace   => $self->framework('ApplicationName'),
+                    compress_threshold => 10240,
+                },
+            },
         }
     };
 

commit 57a4fcc33ad624d6004913cf80e0ee69775e5755
Author: Thomas Sibley <trs at bestpractical.com>
Date:   Tue Jan 26 21:16:53 2010 -0500

    Set an expiration on memcached CAS content

diff --git a/lib/Jifty/CAS/Store/Memcached.pm b/lib/Jifty/CAS/Store/Memcached.pm
index cb5ba8b..d7d5e84 100644
--- a/lib/Jifty/CAS/Store/Memcached.pm
+++ b/lib/Jifty/CAS/Store/Memcached.pm
@@ -51,13 +51,10 @@ Stores the BLOB (a L<Jifty::CAS::Blob>) in memcached.  Returns the key.
 sub _store {
     my ($class, $domain, $name, $blob) = @_;
 
-    # my $db  = $CONTAINER{$domain} ||= {};
-    # $db->{DB}{$key} = $blob;
-    # $db->{KEYS}{$name} = $key;
-
+    # Default to expiring in two weeks. XXX TODO this should be configurable
     my $key = $blob->key;
-    $class->memcached->set("$domain:db:$key", $blob);
-    $class->memcached->set("$domain:keys:$name", $key);
+    $class->memcached->set("$domain:db:$key", $blob, 60*60*24*14);
+    $class->memcached->set("$domain:keys:$name", $key, 60*60*24*14);
 
     return $key;
 }

commit 409707d0b5faa01b126cb26aad4edea883a7c975
Author: Thomas Sibley <trs at bestpractical.com>
Date:   Tue Jan 26 22:18:49 2010 -0500

    Error check the memcached set calls

diff --git a/lib/Jifty/CAS/Store/Memcached.pm b/lib/Jifty/CAS/Store/Memcached.pm
index d7d5e84..9603a0e 100644
--- a/lib/Jifty/CAS/Store/Memcached.pm
+++ b/lib/Jifty/CAS/Store/Memcached.pm
@@ -44,7 +44,8 @@ sub memcached {
 
 =head2 _store DOMAIN NAME BLOB
 
-Stores the BLOB (a L<Jifty::CAS::Blob>) in memcached.  Returns the key.
+Stores the BLOB (a L<Jifty::CAS::Blob>) in memcached.  Returns the key on
+success or undef on failure.
 
 =cut
 
@@ -53,8 +54,25 @@ sub _store {
 
     # Default to expiring in two weeks. XXX TODO this should be configurable
     my $key = $blob->key;
-    $class->memcached->set("$domain:db:$key", $blob, 60*60*24*14);
-    $class->memcached->set("$domain:keys:$name", $key, 60*60*24*14);
+    my $success = $class->memcached->set("$domain:db:$key", $blob, 60*60*24*14);
+
+    unless ($success) {
+        my $err = "Failed to store content for key '$domain:db:$key' in memcached!";
+        {
+            use bytes;
+            $err .= "  Content length is: " . length($blob->content) . " bytes.";
+            $err .= "  Perhaps you need to increase memcached's max item size?";
+        }
+        Jifty->log->error($err);
+        return;
+    }
+
+    $success = $class->memcached->set("$domain:keys:$name", $key, 60*60*24*14);
+
+    unless ($success) {
+        Jifty->log->error("Failed to store key '$domain:keys:$name' in memcached!");
+        return;
+    }
 
     return $key;
 }
diff --git a/lib/Jifty/Config.pm b/lib/Jifty/Config.pm
index f2c4433..fbbf046 100644
--- a/lib/Jifty/Config.pm
+++ b/lib/Jifty/Config.pm
@@ -619,7 +619,7 @@ sub defaults {
                 Memcached => {
                     servers     => [ '127.0.0.1:11211' ],
                     debug       => 0,
-                    namespace   => $self->framework('ApplicationName'),
+                    namespace   => $self->framework('ApplicationName').":",
                     compress_threshold => 10240,
                 },
             },

commit 44359d5a34b95bcd2acf238f6d3eff815c235f03
Author: Thomas Sibley <trs at bestpractical.com>
Date:   Tue Jan 26 22:40:53 2010 -0500

    Support falling back when memcached fails
    
    Fallback to default store, but with an error message.  This may be
    desireable to keep services functioning, but probably should be turned
    off in most cases after successful testing.

diff --git a/lib/Jifty/CAS.pm b/lib/Jifty/CAS.pm
index 63eaaeb..3f7e54f 100644
--- a/lib/Jifty/CAS.pm
+++ b/lib/Jifty/CAS.pm
@@ -45,7 +45,7 @@ however.
 
 Publishes the given C<CONTENT> at the address C<DOMAIN> and C<NAME>.
 C<METADATA> is an arbitrary hash; see L<Jifty::CAS::Blob> for more.
-Returns the key.
+Returns the key on success, or undef on failure.
 
 =head2 key DOMAIN NAME
 
diff --git a/lib/Jifty/CAS/Store/Memcached.pm b/lib/Jifty/CAS/Store/Memcached.pm
index 9603a0e..de429e2 100644
--- a/lib/Jifty/CAS/Store/Memcached.pm
+++ b/lib/Jifty/CAS/Store/Memcached.pm
@@ -64,7 +64,16 @@ sub _store {
             $err .= "  Perhaps you need to increase memcached's max item size?";
         }
         Jifty->log->error($err);
-        return;
+
+        if ( $class->memcached_fallback ) {
+            Jifty->log->error("Falling back to default, in-process memory store.  "
+                             ."This is suboptimal and you should investigate the cause.");
+            return $class->SUPER::_store($domain, $name, $blob);
+        }
+        else {
+            # fail with undef
+            return;
+        }
     }
 
     $success = $class->memcached->set("$domain:keys:$name", $key, 60*60*24*14);
@@ -86,7 +95,10 @@ C<NAME>, or undef if none such exists.
 
 sub key {
     my ($class, $domain, $name) = @_;
-    return $class->memcached->get("$domain:keys:$name");
+    my $key = $class->memcached->get("$domain:keys:$name");
+    return $key if defined $key;
+    return $class->SUPER::key($domain, $name) if $class->memcached_fallback;
+    return;
 }
 
 =head2 retrieve DOMAIN KEY
@@ -98,7 +110,10 @@ C<KEY>, or undef if none such exists.
 
 sub retrieve {
     my ($class, $domain, $key) = @_;
-    return $class->memcached->get("$domain:db:$key");
+    my $blob = $class->memcached->get("$domain:db:$key");
+    return $blob if defined $blob;
+    return $class->SUPER::retrieve($domain, $key) if $class->memcached_fallback;
+    return;
 }
 
 =head2 memcached_config
@@ -132,4 +147,15 @@ sub memcached_config {
         || Jifty->config->defaults->{'framework'}{'CAS'}{'Memcached'}
 }
 
+=head2 memcached_fallback
+
+Returns a boolean (from the config file) indicating whether or not memcached
+should fallback to the per-process, in-memory store.
+
+=cut
+
+sub memcached_fallback {
+    Jifty->config->framework('CAS')->{'MemcachedFallback'} ? 1 : 0
+}
+
 1;
diff --git a/lib/Jifty/Config.pm b/lib/Jifty/Config.pm
index fbbf046..41a7ed3 100644
--- a/lib/Jifty/Config.pm
+++ b/lib/Jifty/Config.pm
@@ -622,6 +622,7 @@ sub defaults {
                     namespace   => $self->framework('ApplicationName').":",
                     compress_threshold => 10240,
                 },
+                MemcachedFallback => 1,
             },
         }
     };

commit 5396e2e35b96b0aebc9f3086b0b90e12f57aeea8
Author: Thomas Sibley <trs at bestpractical.com>
Date:   Tue Jan 26 22:48:40 2010 -0500

    Doc memcached config options

diff --git a/lib/Jifty/CAS/Store/Memcached.pm b/lib/Jifty/CAS/Store/Memcached.pm
index de429e2..b4ca792 100644
--- a/lib/Jifty/CAS/Store/Memcached.pm
+++ b/lib/Jifty/CAS/Store/Memcached.pm
@@ -12,16 +12,33 @@ Content-Addressable Storage facility
 
 =head1 SYNOPSIS
 
-Add the following to your Jifty config.yml:
+At the bare minimum, add the following to your Jifty config.yml:
 
     framework:
       CAS:
-        BaseClass: Jifty::CAS::Store::Memcached
+        BaseClass: 'Jifty::CAS::Store::Memcached'
+
+The options available include:
+
+    framework:
+      CAS:
+        BaseClass: 'Jifty::CAS::Store::Memcached'
+        Memcached:
+          # any options Cache::Memcached supports
+          servers:
+            - 10.0.0.2:11211
+            - 10.0.0.3:11211
+          compress_threshold: 5120
+
+        # Turned on by default. Keeps CAS working when memcached fails by
+        # falling back to the default in-process store. It probably should
+        # be turned off in most cases (like so) after successful testing.
+        MemcachedFallback: 0
 
 =head1 DESCRIPTION
 
-This is a memcached backend for L<Jifty::CAS>.  For more information, see
-L<Jifty::CAS/DESCRIPTION>.
+This is a memcached backend for L<Jifty::CAS>.  For more information about
+Jifty's CAS, see L<Jifty::CAS/DESCRIPTION>.
 
 =cut
 
@@ -133,7 +150,7 @@ C</framework/CAS/Memcached> like so:
 
     framework:
       CAS:
-        BaseClass: Jifty::CAS::Store::Memcached
+        BaseClass: 'Jifty::CAS::Store::Memcached'
         Memcached:
             servers:
                 - 10.0.0.2:11211

commit 9196d3d0661490d6064ec8044f1e06ad8a72b950
Author: Thomas Sibley <trs at bestpractical.com>
Date:   Wed Jan 27 16:11:43 2010 -0500

    Add Cache::Memcached to Makefile.PL

diff --git a/Makefile.PL b/Makefile.PL
index 758618e..aeab27e 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -103,6 +103,11 @@ else {
     requires('YAML' => 0.35) unless can_use('YAML::Syck' => 0.71);
 }
 
+feature "Memcached support for serving compressed CSS and JS from Jifty's CAS" =>
+    -default => 1,
+    recommends('Cache::Memcached' => 1.25),
+    ;
+
 feature 'Administrative Interface (web)' =>
     -default => 1,
     recommends('Pod::Simple' => 0), # Pod::Simple::Text Pod::Simple::HTML

-----------------------------------------------------------------------


More information about the Jifty-commit mailing list