[Jifty-commit] r2182 - in Jifty-DBI/trunk: . lib/Jifty/DBI
jifty-commit at lists.jifty.org
jifty-commit at lists.jifty.org
Mon Nov 20 10:44:12 EST 2006
Author: jesse
Date: Mon Nov 20 10:44:12 2006
New Revision: 2182
Added:
Jifty-DBI/trunk/t/12prefetch.t
Modified:
Jifty-DBI/trunk/ (props changed)
Jifty-DBI/trunk/lib/Jifty/DBI/Collection.pm
Jifty-DBI/trunk/lib/Jifty/DBI/Handle.pm
Jifty-DBI/trunk/lib/Jifty/DBI/Record.pm
Log:
r45209 at pinglin: jesse | 2006-11-20 10:44:00 -0500
* Initial implementation of "Prefetch" of related columns and tables when loading a collection
Modified: Jifty-DBI/trunk/lib/Jifty/DBI/Collection.pm
==============================================================================
--- Jifty-DBI/trunk/lib/Jifty/DBI/Collection.pm (original)
+++ Jifty-DBI/trunk/lib/Jifty/DBI/Collection.pm Mon Nov 20 10:44:12 2006
@@ -54,7 +54,7 @@
use Clone;
use Carp qw/croak/;
use base qw/Class::Accessor::Fast/;
-__PACKAGE__->mk_accessors(qw/pager preload_columns/);
+__PACKAGE__->mk_accessors(qw/pager preload_columns preload_related/);
=head1 METHODS
@@ -101,7 +101,7 @@
@_
);
$self->_handle( $args{'handle'} ) if ( $args{'handle'} );
- $self->table($self->new_item->table());
+ $self->table( $self->new_item->table() );
$self->clean_slate();
}
@@ -193,11 +193,80 @@
my $records = $self->_handle->simple_query($query_string);
return 0 unless $records;
+ my @names = @{ $records->{NAME_lc} };
+ my $data = {};
+ my $column_map = {};
+ foreach my $column (@names) {
+ if ($column =~ /^(.*?)\_(.*)$/) {
+ $column_map->{$1}->{$2} =$column;
+ }
+ }
+ my @tables = keys %$column_map;
+
+
+ my @order;
+ while ( my $base_row = $records->fetchrow_hashref() ) {
+ my $main_pkey = $base_row->{$names[0]};
+ push @order, $main_pkey unless ( $order[0] && $order[-1] eq $main_pkey);
+
+ # let's chop the row into subrows;
+ foreach my $table (@tables) {
+ for ( keys %$base_row ) {
+ if ( $_ =~ /$table\_(.*)$/ ) {
+ $data->{$main_pkey}->{$table} ->{ ($base_row->{ $table . '_id' } ||$main_pkey )}->{$1} = $base_row->{$_};
+ }
+ }
+ }
+
+ }
+
+ # For related "record" values, we can simply prepopulate the
+ # Jifty::DBI::Record cache and life will be good. (I suspect we want
+ # to do this _before_ doing the initial primary record load on the
+ # off chance that the primary record will try to do the relevant
+ # prefetch manually For related "collection" values, our job is a bit
+ # harder. we need to create a new empty collection object, set it's
+ # "must search" to 0 and manually add the records to it for each of
+ # the items we find. Then we need to ram it into place.
+
+ foreach my $row_id ( @order) {
+ my $item;
+ foreach my $row ( values %{ $data->{$row_id}->{'main'} } ) {
+ $item = $self->new_item();
+ $item->load_from_hash($row);
+ }
+ foreach my $alias ( grep { $_ ne 'main' } keys %{ $data->{$row_id} } ) {
+
+ my $related_rows = $data->{$row_id}->{$alias};
+ my ( $class, $col_name ) = $self->class_and_column_for_alias($alias);
+ if ($class) {
+
+ if ( $class->isa('Jifty::DBI::Collection') ) {
+ my $collection = $class->new( handle => $self->_handle );
+ foreach my $row( sort { $a->{id} <=> $b->{id} } values %$related_rows ) {
+ my $entry
+ = $collection->new_item( handle => $self->_handle );
+ $entry->load_from_hash($row);
+ $collection->add_record($entry);
+ }
+
+ $item->_prefetched_collection( $col_name => $collection );
+ } elsif ( $class->isa('Jifty::DBI::Record') ) {
+ foreach my $related_row ( values %$related_rows ) {
+ my $item = $class->new( handle => $self->_handle );
+ $item->load_from_hash($related_row);
+ }
+ } else {
+ Carp::cluck(
+ "Asked to preload $alias as a $class. Don't know how to handle $class"
+ );
+ }
+ }
- while ( my $row = $records->fetchrow_hashref() ) {
- my $item = $self->new_item();
- $item->load_from_hash($row);
+
+ }
$self->add_record($item);
+
}
if ( $records->err ) {
$self->{'must_redo_search'} = 0;
@@ -278,10 +347,6 @@
$self->_handle->apply_limits( $statementref, $self->rows_per_page,
$self->first_row );
-# XXX TODO: refactor me, once we figure out the last place that columns could be set
- $$statementref =~ s/main\.\*/CORE::join(', ', @{$self->{columns}})/eg
- if $self->{columns}
- and @{ $self->{columns} };
}
=head2 _distinct_query STATEMENTREF
@@ -384,7 +449,8 @@
# DISTINCT query only required for multi-table selects
$self->_distinct_query( \$query_string );
} else {
- $query_string = "SELECT ".$self->_preload_columns." FROM $query_string";
+ $query_string
+ = "SELECT " . $self->_preload_columns . " FROM $query_string";
$query_string .= $self->_group_clause;
$query_string .= $self->_order_clause;
}
@@ -405,8 +471,106 @@
sub _preload_columns {
my $self = shift;
- return 'main.*' unless $self->preload_columns;
- return join(',', map { "main.$_" } @{$self->preload_columns});
+
+ my @cols = ();
+ my $item = $self->new_item;
+ if( $self->{columns} and @{ $self->{columns} } ) {
+ push @cols, @{$self->{columns}};
+ # push @cols, map { warn "Preloading $_"; "main.$_ as main_" . $_ } @{$preload_columns};
+ } else {
+ push @cols, $self->_qualified_record_columns( 'main' => $item );
+ }
+ my %preload_related = %{ $self->preload_related || {} };
+ foreach my $alias ( keys %preload_related ) {
+ my $related_obj = $preload_related{$alias};
+ if ( my $col_obj = $item->column($related_obj) ) {
+ my $reference_type = $col_obj->refers_to;
+
+ my $reference_item;
+
+ if ( !$reference_type ) {
+ Carp::cluck(
+ "Asked to prefetch $col_obj->name for $self. But $col_obj->name isn't a known reference"
+ );
+ } elsif ( $reference_type->isa('Jifty::DBI::Collection') ) {
+ $reference_item = $reference_type->new->new_item();
+ } elsif ( $reference_type->isa('Jifty::DBI::Record') ) {
+ $reference_item = $reference_type->new;
+ } else {
+ Carp::cluck(
+ "Asked to prefetch $col_obj->name for $self. But $col_obj->name isn't a known type"
+ );
+ }
+
+ push @cols,
+ $self->_qualified_record_columns( $alias => $reference_item );
+ }
+
+ # push @cols, map { $_ . ".*" } keys %{ $self->preload_related || {} };
+
+ }
+ return join( ', ', @cols );
+}
+
+=head2 class_and_column_for_alias
+
+Takes the alias you've assigned to a prefetched related object. Returns the class
+of the column we've declared that alias preloads.
+
+=cut
+
+sub class_and_column_for_alias {
+ my $self = shift;
+ my $alias = shift;
+ my %preload_related = %{ $self->preload_related || {} };
+ my $related_colname = $preload_related{$alias};
+ if ( my $col_obj = $self->new_item->column($related_colname) ) {
+ return ( $col_obj->refers_to => $related_colname );
+ }
+ return undef;
+}
+
+sub _qualified_record_columns {
+ my $self = shift;
+ my $alias = shift;
+ my $item = shift;
+ grep {$_} map {
+ my $col = $_;
+ if ( $col->virtual ) {
+ undef;
+ } else {
+ $col = $col->name;
+ $alias . "." . $col . " as " . $alias . "_" . $col;
+ }
+ } $item->columns;
+}
+
+=head2 prefetch ALIAS_NAME ATTRIBUTE
+
+prefetches all related rows from alias ALIAS_NAME into the record attribute ATTRIBUTE of the
+sort of item this collection is.
+
+If you have employees who have many phone numbers, this method will let you search for all your employees
+ and prepopulate their phone numbers.
+
+Right now, in order to make this work, you need to do an explicit join between your primary table and the subsidiary tables AND then specify the name of the attribute you want to prefetch related data into.
+This method could be a LOT smarter. since we already know what the relationships between our tables are, that could all be precomputed.
+
+XXX TODO: in the future, this API should be extended to let you specify columns.
+
+=cut
+
+sub prefetch {
+ my $self = shift;
+ my $alias = shift;
+ my $into_attribute = shift;
+
+ my $preload_related = $self->preload_related() || {};
+
+ $preload_related->{$alias} = $into_attribute;
+
+ $self->preload_related($preload_related);
+
}
=head2 distinct_required
@@ -421,7 +585,6 @@
=cut
-
sub distinct_required {
my $self = shift;
return $self->_is_joined ? 1 : 0;
@@ -735,7 +898,7 @@
unless defined $args{value};
# make passing in an object DTRT
- if (ref($args{value}) && $args{value}->isa('Jifty::DBI::Record')) {
+ if ( ref( $args{value} ) && $args{value}->isa('Jifty::DBI::Record') ) {
$args{value} = $args{value}->id;
}
@@ -838,18 +1001,19 @@
# If it's a new value or we're overwriting this sort of restriction,
- if ($self->_handle->case_sensitive
+ if ( $self->_handle->case_sensitive
&& defined $args{'value'}
&& $args{'quote_value'}
- && ! $args{'case_sensitive'}) {
- # don't worry about case for numeric columns_in_db
- my $column_obj = $self->new_item()->column($args{column});
- if (defined $column_obj ? ! $column_obj->is_numeric : 1) {
- ( $qualified_column, $args{'operator'}, $args{'value'} ) =
- $self->_handle->_make_clause_case_insensitive( $qualified_column,
- $args{'operator'},
- $args{'value'} );
- }
+ && !$args{'case_sensitive'} )
+ {
+
+ # don't worry about case for numeric columns_in_db
+ my $column_obj = $self->new_item()->column( $args{column} );
+ if ( defined $column_obj ? !$column_obj->is_numeric : 1 ) {
+ ( $qualified_column, $args{'operator'}, $args{'value'} )
+ = $self->_handle->_make_clause_case_insensitive(
+ $qualified_column, $args{'operator'}, $args{'value'} );
+ }
}
my $clause = "($qualified_column $args{'operator'} $args{'value'})";
@@ -948,7 +1112,8 @@
push @subclauses, $self->{'subclauses'}{"$subclause"};
}
- $where_clause = " WHERE " . CORE::join( ' AND ', @subclauses ) if (@subclauses);
+ $where_clause = " WHERE " . CORE::join( ' AND ', @subclauses )
+ if (@subclauses);
return ($where_clause);
@@ -1020,17 +1185,17 @@
=cut
sub order_by {
- my $self = shift;
- if (@_) {
- my @args = @_;
+ my $self = shift;
+ if (@_) {
+ my @args = @_;
- unless ( UNIVERSAL::isa( $args[0], 'HASH' ) ) {
- @args = {@args};
+ unless ( UNIVERSAL::isa( $args[0], 'HASH' ) ) {
+ @args = {@args};
+ }
+ $self->{'order_by'} = \@args;
+ $self->redo_search();
}
- $self->{'order_by'} = \@args;
- $self->redo_search();
- }
- return $self->{'order_by'};
+ return $self->{'order_by'};
}
=head2 _order_clause
@@ -1270,7 +1435,7 @@
->current_page( $args{'current_page'} );
$self->rows_per_page( $args{'per_page'} );
- $self->first_row( $self->pager->first ||1 );
+ $self->first_row( $self->pager->first || 1 );
}
@@ -1482,6 +1647,7 @@
my $column = "col" . @{ $self->{columns} ||= [] };
$column = $args{column} if $table eq $self->table and !$args{alias};
+ $column = ($args{'alias'}||'main')."_".$column;
push @{ $self->{columns} }, "$name AS \L$column";
return $column;
}
@@ -1565,7 +1731,7 @@
sub refers_to {
my $class = shift;
- return ( Jifty::DBI::Schema::Trait->new(refers_to => $class), @_ );
+ return ( Jifty::DBI::Schema::Trait->new( refers_to => $class ), @_ );
}
=head2 clone
Modified: Jifty-DBI/trunk/lib/Jifty/DBI/Handle.pm
==============================================================================
--- Jifty-DBI/trunk/lib/Jifty/DBI/Handle.pm (original)
+++ Jifty-DBI/trunk/lib/Jifty/DBI/Handle.pm Mon Nov 20 10:44:12 2006
@@ -275,7 +275,7 @@
=head2 sql_statement_log
-Returns the current SQL statement log as an array of arrays. Each entry is a triple of
+Returns the current SQL statement log as an array of arrays. Each entry is a list of
(Time, Statement, [Bindings], Duration)
@@ -984,7 +984,7 @@
my $sb = shift;
# Prepend select query for DBs which allow DISTINCT on all column types.
- $$statementref = "SELECT DISTINCT main.* FROM $$statementref";
+ $$statementref = "SELECT DISTINCT ".$sb->_preload_columns." FROM $$statementref";
$$statementref .= $sb->_group_clause;
$$statementref .= $sb->_order_clause;
Modified: Jifty-DBI/trunk/lib/Jifty/DBI/Record.pm
==============================================================================
--- Jifty-DBI/trunk/lib/Jifty/DBI/Record.pm (original)
+++ Jifty-DBI/trunk/lib/Jifty/DBI/Record.pm Mon Nov 20 10:44:12 2006
@@ -324,11 +324,31 @@
return undef unless $classname;
return unless UNIVERSAL::isa( $classname, 'Jifty::DBI::Collection' );
+ if ( my $prefetched_col = $self->_prefetched_collection($method_name)) {
+ warn "We have a preetch for $method_name";
+ return $prefetched_col;
+ }
+
+ warn "no prefetch";
my $coll = $classname->new( handle => $self->_handle );
$coll->limit( column => $column->by(), value => $self->id );
return $coll;
}
+sub _prefetched_collection {
+ my $self =shift;
+ my $column_name = shift;
+ if (@_) {
+ warn "Setting up a prefetch collection for $column_name";
+ $self->{'_prefetched_collections'}->{$column_name} = shift;
+ } else {
+ warn "We'll load a prefetch collection for $column_name";
+ return $self->{'_prefetched_collections'}->{$column_name};
+ }
+
+}
+
+
=head2 add_column
=cut
Added: Jifty-DBI/trunk/t/12prefetch.t
==============================================================================
--- (empty file)
+++ Jifty-DBI/trunk/t/12prefetch.t Mon Nov 20 10:44:12 2006
@@ -0,0 +1,206 @@
+#!/usr/bin/env perl -w
+
+
+use strict;
+use warnings;
+use File::Spec;
+use Test::More 'no_plan';
+
+BEGIN { require "t/utils.pl" }
+our (@available_drivers);
+
+use constant TESTS_PER_DRIVER => 67;
+
+my $total = scalar(@available_drivers) * TESTS_PER_DRIVER;
+#plan tests => $total;
+
+foreach my $d ('SQLite'){ # @available_drivers ) {
+SKIP: {
+ unless( has_schema( 'TestApp', $d ) ) {
+ skip "No schema for '$d' driver", TESTS_PER_DRIVER;
+ }
+ unless( should_test( $d ) ) {
+ skip "ENV is not defined for driver '$d'", TESTS_PER_DRIVER;
+ }
+
+ my $handle = get_handle( $d );
+ connect_handle( $handle );
+ isa_ok($handle->dbh, 'DBI::db', "Got handle for $d");
+
+ {my $ret = init_schema( 'TestApp', $handle );
+ isa_ok($ret,'DBI::st', "Inserted the schema. got a statement handle back" );}
+
+
+ my $emp = TestApp::Employee->new( handle => $handle );
+ my $e_id = $emp->create( Name => 'RUZ' );
+ ok($e_id, "Got an id for the new employee: $e_id");
+ $emp->load($e_id);
+ is($emp->id, $e_id);
+
+ my $phone_collection = $emp->phones;
+ isa_ok($phone_collection, 'TestApp::PhoneCollection');
+ {
+ my $phone = TestApp::Phone->new( handle => $handle );
+ isa_ok( $phone, 'TestApp::Phone');
+ my $p_id = $phone->create( employee => $e_id, phone => '+7(903)264-03-51');
+ is($p_id, 1, "Loaded phone $p_id");
+ $phone->load( $p_id );
+
+ my $obj = $phone->employee;
+
+ ok($obj, "Employee #$e_id has phone #$p_id");
+ isa_ok( $obj, 'TestApp::Employee');
+ is($obj->id, $e_id);
+ is($obj->name, 'RUZ');
+ }
+
+ my $emp2 = TestApp::Employee->new( handle => $handle );
+ my $e2_id = $emp2->create( Name => 'JESSE' );
+ my $phone2 = TestApp::Phone->new( handle => $handle );
+ my $p2_id = $phone2->create( employee => $e2_id, phone => '+16173185823');
+
+ for (3..6){
+ my $i = $_;
+ diag("loading phone $i");
+ my $phone = TestApp::Phone->new( handle => $handle );
+ isa_ok( $phone, 'TestApp::Phone');
+ my $p_id = $phone->create( employee => $e_id, phone => "+1 $i");
+ is($p_id, $i, "Loaded phone $p_id");
+ $phone->load( $p_id );
+
+ my $obj = $phone->employee;
+
+ ok($obj, "Employee #$e_id has phone #$p_id");
+ isa_ok( $obj, 'TestApp::Employee');
+ is($obj->id, $e_id);
+ is($obj->name, 'RUZ');
+
+
+ }
+
+ $handle->log_sql_statements(1);
+
+ my $collection
+ = TestApp::EmployeeCollection->new( handle => $handle );
+ my $phones_alias = $collection->join(
+ alias1 => 'main',
+ column1 => 'id',
+ table2 => 'phones',
+ column2 => 'employee'
+ );
+ $collection->prefetch( $phones_alias => 'phones'); #
+ $collection->limit( column => 'id', value => '1', operator => '>=' );
+ my $user = $collection->next;
+ is( $user->id, 1, "got our user" );
+ my $phones = $user->phones;
+ is( $phones->first->id, 1 );
+ is( $phones->count, 5 );
+
+
+ my $jesse = $collection->next;
+ is ($jesse->name, 'JESSE');
+ my $jphone = $jesse->phones;
+ is ($jphone->count,1);
+
+ my @statements = $handle->sql_statement_log;
+
+ is (scalar @statements, 1, "all that. just one sql statement");
+
+# cleanup_schema( 'TestApp', $handle );
+ disconnect_handle( $handle );
+}} # SKIP, foreach blocks
+
+1;
+
+
+package TestApp;
+sub schema_sqlite {
+[
+q{
+CREATE table employees (
+ id integer primary key,
+ name varchar(36)
+)
+}, q{
+CREATE table phones (
+ id integer primary key,
+ employee integer NOT NULL,
+ phone varchar(18)
+) }
+]
+}
+
+sub schema_mysql {
+[ q{
+CREATE TEMPORARY table employees (
+ id integer AUTO_INCREMENT primary key,
+ name varchar(36)
+)
+}, q{
+CREATE TEMPORARY table phones (
+ id integer AUTO_INCREMENT primary key,
+ employee integer NOT NULL,
+ phone varchar(18)
+)
+} ]
+}
+
+sub schema_pg {
+[ q{
+CREATE TEMPORARY table employees (
+ id serial PRIMARY KEY,
+ name varchar
+)
+}, q{
+CREATE TEMPORARY table phones (
+ id serial PRIMARY KEY,
+ employee integer references employees(id),
+ phone varchar
+)
+} ]
+}
+
+package TestApp::Employee;
+use base qw/Jifty::DBI::Record/;
+
+sub _value {
+ my $self = shift;
+ my $x = ($self->__value(@_));
+ return $x;
+}
+
+package TestApp::Phone;
+use base qw/Jifty::DBI::Record/;
+
+package TestApp::PhoneCollection;
+use base qw/Jifty::DBI::Collection/;
+
+sub table {
+ my $self = shift;
+ my $tab = $self->new_item->table();
+ return $tab;
+}
+
+
+package TestApp::Phone::Schema;
+BEGIN {
+ use Jifty::DBI::Schema;
+ column employee => refers_to TestApp::Employee;
+ column phone => type 'varchar';
+}
+
+package TestApp::Employee::Schema;
+BEGIN {
+ use Jifty::DBI::Schema;
+ column name => type 'varchar';
+ column phones => refers_to TestApp::PhoneCollection by 'employee';
+}
+
+
+package TestApp::EmployeeCollection;
+
+use base qw/Jifty::DBI::Collection/;
+
+
+
+1;
More information about the Jifty-commit
mailing list