Filename | /usr/local/src/github.com/foswiki/core/lib/Foswiki/Store/QueryAlgorithms/BruteForce.pm |
Statements | Executed 171 statements in 3.17ms |
Calls | P | F | Exclusive Time |
Inclusive Time |
Subroutine |
---|---|---|---|---|---|
1 | 1 | 1 | 3.99ms | 10.8ms | BEGIN@29 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 1.17ms | 1.65s | _webQuery | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 46µs | 59µs | BEGIN@26 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 36µs | 55µs | new | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 28µs | 61µs | BEGIN@27 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 16µs | 113µs | BEGIN@46 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 10µs | 10µs | BEGIN@32 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 10µs | 10µs | BEGIN@33 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 10µs | 10µs | BEGIN@34 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 10µs | 10µs | BEGIN@43 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 9µs | 9µs | BEGIN@41 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 9µs | 9µs | BEGIN@35 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 9µs | 9µs | BEGIN@37 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 9µs | 9µs | BEGIN@40 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 9µs | 9µs | BEGIN@36 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 9µs | 9µs | BEGIN@44 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 9µs | 9µs | BEGIN@42 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 9µs | 9µs | BEGIN@38 | Foswiki::Store::QueryAlgorithms::BruteForce::
1 | 1 | 1 | 9µs | 9µs | BEGIN@39 | Foswiki::Store::QueryAlgorithms::BruteForce::
Line | State ments |
Time on line |
Calls | Time in subs |
Code |
---|---|---|---|---|---|
1 | # See bottom of file for license and copyright information | ||||
2 | package Foswiki::Store::QueryAlgorithms::BruteForce; | ||||
3 | |||||
4 | =begin TML | ||||
5 | |||||
6 | ---+ package Foswiki::Store::QueryAlgorithms::BruteForce | ||||
7 | Implements Foswiki::Store::Interfaces::QueryAlgorithm | ||||
8 | |||||
9 | Default brute-force query algorithm. Works by hoisting regular expressions | ||||
10 | out of the query tree to narrow down the set of topics to be tested. Then | ||||
11 | uses the query 'evaluate' method on each topic in turn to fully evaluate | ||||
12 | the remaining query. | ||||
13 | |||||
14 | Not sure exactly where the breakpoint is between the | ||||
15 | costs of hoisting and the advantages of hoisting. Benchmarks suggest | ||||
16 | that it's around 6 topics, though this may vary depending on disk | ||||
17 | speed and memory size. It also depends on the complexity of the query. | ||||
18 | |||||
19 | =cut | ||||
20 | |||||
21 | # TODO: There is an additional opportunity for optimisation; if we assume | ||||
22 | # the grep is solid, we can cut those parts of the query out for the full | ||||
23 | # evaluation path. Not done yet, because CDot strongly suspects it won't make | ||||
24 | # much difference. | ||||
25 | |||||
26 | 2 | 77µs | 2 | 72µs | # spent 59µs (46+13) within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@26 which was called:
# once (46µs+13µs) by Foswiki::Store::VC::Store::query at line 26 # spent 59µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@26
# spent 13µs making 1 call to strict::import |
27 | 2 | 62µs | 2 | 95µs | # spent 61µs (28+34) within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@27 which was called:
# once (28µs+34µs) by Foswiki::Store::VC::Store::query at line 27 # spent 61µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@27
# spent 34µs making 1 call to warnings::import |
28 | |||||
29 | 2 | 186µs | 1 | 10.8ms | # spent 10.8ms (3.99+6.82) within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@29 which was called:
# once (3.99ms+6.82ms) by Foswiki::Store::VC::Store::query at line 29 # spent 10.8ms making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@29 |
30 | 1 | 10µs | our @ISA = ('Foswiki::Store::Interfaces::QueryAlgorithm'); | ||
31 | |||||
32 | 2 | 43µs | 1 | 10µs | # spent 10µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@32 which was called:
# once (10µs+0s) by Foswiki::Store::VC::Store::query at line 32 # spent 10µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@32 |
33 | 2 | 40µs | 1 | 10µs | # spent 10µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@33 which was called:
# once (10µs+0s) by Foswiki::Store::VC::Store::query at line 33 # spent 10µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@33 |
34 | 2 | 39µs | 1 | 10µs | # spent 10µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@34 which was called:
# once (10µs+0s) by Foswiki::Store::VC::Store::query at line 34 # spent 10µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@34 |
35 | 2 | 40µs | 1 | 9µs | # spent 9µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@35 which was called:
# once (9µs+0s) by Foswiki::Store::VC::Store::query at line 35 # spent 9µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@35 |
36 | 2 | 38µs | 1 | 9µs | # spent 9µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@36 which was called:
# once (9µs+0s) by Foswiki::Store::VC::Store::query at line 36 # spent 9µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@36 |
37 | 2 | 38µs | 1 | 9µs | # spent 9µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@37 which was called:
# once (9µs+0s) by Foswiki::Store::VC::Store::query at line 37 # spent 9µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@37 |
38 | 2 | 38µs | 1 | 9µs | # spent 9µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@38 which was called:
# once (9µs+0s) by Foswiki::Store::VC::Store::query at line 38 # spent 9µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@38 |
39 | 2 | 38µs | 1 | 9µs | # spent 9µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@39 which was called:
# once (9µs+0s) by Foswiki::Store::VC::Store::query at line 39 # spent 9µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@39 |
40 | 2 | 38µs | 1 | 9µs | # spent 9µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@40 which was called:
# once (9µs+0s) by Foswiki::Store::VC::Store::query at line 40 # spent 9µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@40 |
41 | 2 | 38µs | 1 | 9µs | # spent 9µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@41 which was called:
# once (9µs+0s) by Foswiki::Store::VC::Store::query at line 41 # spent 9µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@41 |
42 | 2 | 38µs | 1 | 9µs | # spent 9µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@42 which was called:
# once (9µs+0s) by Foswiki::Store::VC::Store::query at line 42 # spent 9µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@42 |
43 | 2 | 39µs | 1 | 10µs | # spent 10µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@43 which was called:
# once (10µs+0s) by Foswiki::Store::VC::Store::query at line 43 # spent 10µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@43 |
44 | 2 | 45µs | 1 | 9µs | # spent 9µs within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@44 which was called:
# once (9µs+0s) by Foswiki::Store::VC::Store::query at line 44 # spent 9µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@44 |
45 | |||||
46 | 2 | 1.24ms | 2 | 210µs | # spent 113µs (16+97) within Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@46 which was called:
# once (16µs+97µs) by Foswiki::Store::VC::Store::query at line 46 # spent 113µs making 1 call to Foswiki::Store::QueryAlgorithms::BruteForce::BEGIN@46
# spent 97µs making 1 call to constant::import |
47 | |||||
48 | =begin TML | ||||
49 | |||||
50 | ---++ ClassMethod new( $class, ) -> $cereal | ||||
51 | |||||
52 | =cut | ||||
53 | |||||
54 | # spent 55µs (36+19) within Foswiki::Store::QueryAlgorithms::BruteForce::new which was called:
# once (36µs+19µs) by Foswiki::Store::VC::Store::query at line 557 of /usr/local/src/github.com/foswiki/core/lib/Foswiki/Store/VC/Store.pm | ||||
55 | 2 | 33µs | 1 | 19µs | my $self = shift()->SUPER::new( 'SEARCH', @_ ); # spent 19µs making 1 call to Foswiki::Store::Interfaces::QueryAlgorithm::new |
56 | return $self; | ||||
57 | } | ||||
58 | |||||
59 | # Query over a single web | ||||
60 | # spent 1.65s (1.17ms+1.65) within Foswiki::Store::QueryAlgorithms::BruteForce::_webQuery which was called:
# once (1.17ms+1.65s) by Foswiki::Store::Interfaces::QueryAlgorithm::__ANON__[/usr/local/src/github.com/foswiki/core/lib/Foswiki/Store/Interfaces/QueryAlgorithm.pm:109] at line 99 of /usr/local/src/github.com/foswiki/core/lib/Foswiki/Store/Interfaces/QueryAlgorithm.pm | ||||
61 | 16 | 341µs | my ( $this, $query, $web, $inputTopicSet, $session, $options ) = @_; | ||
62 | |||||
63 | 1 | 99µs | my $resultTopicSet = # spent 99µs making 1 call to Foswiki::Search::InfoCache::new | ||
64 | Foswiki::Search::InfoCache->new( $Foswiki::Plugins::SESSION, $web ); | ||||
65 | |||||
66 | # see if this query can be fasttracked. | ||||
67 | # TODO: is this simplification call appropriate here, or should it | ||||
68 | # go in Search.pm | ||||
69 | # TODO: what about simplify to constant in _this_ web? | ||||
70 | my $queryIsAConstantFastpath; # undefined if this is a 'real' query' | ||||
71 | 1 | 51µs | my $context = Foswiki::Meta->new( $session, $session->{webName} ); # spent 51µs making 1 call to Foswiki::Meta::new | ||
72 | 1 | 111µs | $query->simplify( tom => $context, data => $context ); # spent 111µs making 1 call to Foswiki::Query::Node::simplify | ||
73 | |||||
74 | 3 | 45µs | 1 | 11µs | if ( $query->evaluatesToConstant() ) { # spent 11µs making 1 call to Foswiki::Query::Node::evaluatesToConstant |
75 | print STDERR "-- constant?\n" if MONITOR; | ||||
76 | |||||
77 | # SMELL: use any old topic | ||||
78 | 3 | 1.18s | my $cache = $Foswiki::Plugins::SESSION->search->metacache->get( $web, # spent 1.18s making 1 call to Foswiki::MetaCache::get
# spent 11µs making 1 call to Foswiki::Search::metacache
# spent 8µs making 1 call to Foswiki::search | ||
79 | 'WebPreferences' ); | ||||
80 | my $meta = $cache->{tom}; | ||||
81 | 1 | 63µs | $queryIsAConstantFastpath = # spent 63µs making 1 call to Foswiki::Query::Node::evaluate | ||
82 | $query->evaluate( tom => $meta, data => $meta ); | ||||
83 | } | ||||
84 | |||||
85 | 1 | 1µs | if ( defined($queryIsAConstantFastpath) ) { | ||
86 | if ( not $queryIsAConstantFastpath ) { | ||||
87 | print STDERR "-- no results\n" if MONITOR; | ||||
88 | |||||
89 | #CONSTANT _and_ FALSE - return no results | ||||
90 | return $resultTopicSet; | ||||
91 | } | ||||
92 | } | ||||
93 | else { | ||||
94 | print STDERR "-- not constant\n" if MONITOR; | ||||
95 | |||||
96 | # from here on, FALSE means its not a constant, TRUE | ||||
97 | # means is is a constant and evals to TRUE | ||||
98 | $queryIsAConstantFastpath = 0; | ||||
99 | } | ||||
100 | |||||
101 | # Try and hoist regular expressions out of the query that we | ||||
102 | # can use to refine the topic set | ||||
103 | |||||
104 | 1 | 43µs | my $hoistedREs = Foswiki::Query::HoistREs::hoist($query); # spent 43µs making 1 call to Foswiki::Query::HoistREs::hoist | ||
105 | print STDERR "-- hoisted " . Data::Dumper->Dump( [$hoistedREs] ) . "\n" | ||||
106 | if MONITOR; | ||||
107 | |||||
108 | # Reduce the input topic set by matching simple topic names hoisted | ||||
109 | # from the query. | ||||
110 | |||||
111 | if ( ( !defined( $options->{topic} ) ) | ||||
112 | and ( $hoistedREs->{name} ) | ||||
113 | and ( scalar( @{ $hoistedREs->{name} } ) == 1 ) ) | ||||
114 | { | ||||
115 | |||||
116 | # only do this if the 'name' query is simple | ||||
117 | # (ie, has only one element) | ||||
118 | my @filter = @{ $hoistedREs->{name_source} }; | ||||
119 | |||||
120 | #set the 'includetopic' matcher.. | ||||
121 | $options->{topic} = $filter[0]; | ||||
122 | } | ||||
123 | |||||
124 | # Reduce the input topic set by matching the hoisted REs against | ||||
125 | # the topics in it. | ||||
126 | |||||
127 | my $topicSet = $inputTopicSet; | ||||
128 | 2 | 38µs | if ( !defined($topicSet) ) { | ||
129 | print STDERR "-- new topic Set from $web\n" if MONITOR; | ||||
130 | |||||
131 | # then we start with the whole web? | ||||
132 | # TODO: i'm sure that is a flawed assumption | ||||
133 | 1 | 86µs | my $webObject = Foswiki::Meta->new( $session, $web ); # spent 86µs making 1 call to Foswiki::Meta::new | ||
134 | 1 | 170ms | $topicSet = # spent 170ms making 1 call to Foswiki::Search::InfoCache::getTopicListIterator | ||
135 | Foswiki::Search::InfoCache::getTopicListIterator( $webObject, | ||||
136 | $options ); | ||||
137 | } | ||||
138 | |||||
139 | # TODO: how to ask iterator for list length? | ||||
140 | # TODO: once the inputTopicSet isa ResultSet we might have an idea | ||||
141 | # TODO: I presume $hoisetedRE's is undefined for constant queries.. | ||||
142 | # if (() and ( scalar(@$topics) > 6 )) { | ||||
143 | 1 | 1µs | if ( defined( $hoistedREs->{text} ) ) { | ||
144 | my $searchOptions = { | ||||
145 | type => 'regex', | ||||
146 | casesensitive => 1, | ||||
147 | files_without_match => 1, | ||||
148 | web => $web, | ||||
149 | }; | ||||
150 | my @filter = @{ $hoistedREs->{text} }; | ||||
151 | my $searchQuery = | ||||
152 | Foswiki::Search::Node->new( $query->toString(), \@filter, | ||||
153 | $searchOptions ); | ||||
154 | |||||
155 | #use Data::Dumper; | ||||
156 | #print STDERR "--- hoisted: ".Dumper($hoistedREs)."\n" if MONITOR; | ||||
157 | |||||
158 | $topicSet->reset(); | ||||
159 | $topicSet = | ||||
160 | $session->{store} | ||||
161 | ->query( $searchQuery, $topicSet, $session, $searchOptions ); | ||||
162 | } | ||||
163 | else { | ||||
164 | |||||
165 | # TODO: clearly _this_ can be re-written as a FilterIterator, | ||||
166 | # and if we are able to use the sorting hints (ie DB Store) | ||||
167 | # can propogate all the way to FORMAT | ||||
168 | |||||
169 | print STDERR "WARNING: couldn't hoistREs on " . $query->toString() | ||||
170 | if MONITOR; | ||||
171 | } | ||||
172 | |||||
173 | local $/; | ||||
174 | 1 | 40µs | $topicSet->reset(); # spent 40µs making 1 call to Foswiki::Iterator::FilterIterator::reset | ||
175 | 1 | 317µs | while ( $topicSet->hasNext() ) { # spent 317µs making 1 call to Foswiki::Iterator::FilterIterator::hasNext | ||
176 | 66 | 458µs | 22 | 416µs | my $webtopic = $topicSet->next(); # spent 416µs making 22 calls to Foswiki::Iterator::FilterIterator::next, avg 19µs/call |
177 | 22 | 1.34ms | my ( $Iweb, $topic ) = # spent 1.34ms making 22 calls to Foswiki::Func::normalizeWebTopicName, avg 61µs/call | ||
178 | Foswiki::Func::normalizeWebTopicName( $web, $webtopic ); | ||||
179 | print STDERR "-- $Iweb, $topic\n" if MONITOR; | ||||
180 | |||||
181 | 44 | 162µs | 22 | 292ms | if ($queryIsAConstantFastpath) { # spent 292ms making 22 calls to Foswiki::Iterator::FilterIterator::hasNext, avg 13.3ms/call |
182 | print STDERR "-- add $Iweb, $topic\n" if MONITOR; | ||||
183 | if ( defined( $options->{date} ) ) { | ||||
184 | |||||
185 | # TODO: preload the meta cache if we're doing date | ||||
186 | # based filtering - else the wrong filedate will be used | ||||
187 | $Foswiki::Plugins::SESSION->search->metacache->get( $Iweb, | ||||
188 | $topic ); | ||||
189 | } | ||||
190 | |||||
191 | # TODO: frustratingly, there is no way to evaluate a | ||||
192 | # filterIterator without actually iterating over it.. | ||||
193 | 22 | 2.20ms | $resultTopicSet->addTopics( $Iweb, $topic ); # spent 2.20ms making 22 calls to Foswiki::Search::InfoCache::addTopics, avg 100µs/call | ||
194 | } | ||||
195 | else { | ||||
196 | my $meta = | ||||
197 | $Foswiki::Plugins::SESSION->search->metacache->addMeta( $Iweb, | ||||
198 | $topic ); | ||||
199 | print STDERR "-- evaluate $Iweb, $topic\n" if MONITOR; | ||||
200 | next unless ( defined($meta) ); #not a valid or loadable topic | ||||
201 | |||||
202 | # this 'lazy load' will become useful when @$topics becomes | ||||
203 | # an infoCache | ||||
204 | $meta = $meta->load() unless ( $meta->latestIsLoaded() ); | ||||
205 | print STDERR "Processing $topic\n" | ||||
206 | if Foswiki::Query::Node::MONITOR_EVAL; | ||||
207 | my $match = $query->evaluate( tom => $meta, data => $meta ); | ||||
208 | if ($match) { | ||||
209 | $resultTopicSet->addTopic($meta); | ||||
210 | } | ||||
211 | } | ||||
212 | } | ||||
213 | |||||
214 | return $resultTopicSet; | ||||
215 | } | ||||
216 | |||||
217 | 1 | 6µs | 1; | ||
218 | __END__ |