[Bio] / FigKernelPackages / representative_sequences.pm Repository:
ViewVC logotype

Annotation of /FigKernelPackages/representative_sequences.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.1 - (view) (download) (as text)

1 : golsen 1.1 package representative_sequences;
2 :    
3 :     use strict;
4 :     use gjoparseblast;
5 :    
6 :     # These four variables are used as globals below. They would be changed
7 :     # for the SEED environment. formatdb is run as a system command, and might
8 :     # be encapsulated in a run() construct for the SEED. blastall is in an
9 :     # input pipe.
10 :    
11 :     my $tmp_dir = ".";
12 :     my $formatdb = "formatdb";
13 :     my $blastall = "blastall";
14 :    
15 :     require Exporter;
16 :    
17 :     our @ISA = qw(Exporter);
18 :     our @EXPORT = qw( representative_sequences
19 :     rep_seq_2
20 :     );
21 :    
22 :     #===============================================================================
23 :     # Construct a representative set of related sequences:
24 :     #
25 :     # \@repseqs = representative_sequences( $ref, \@seqs, $max_sim, \%options );
26 :     #
27 :     # or
28 :     #
29 :     # ( \@repseqs, \%representing, \@low_sim ) = representative_sequences( $ref,
30 :     # \@seqs, $max_sim, \%options );
31 :     #
32 :     #
33 :     # Output:
34 :     #
35 :     # \@repseqs Reference to the list of retained (representative subset)
36 :     # of sequence entries. Sequence entries have the form
37 :     # [ $id, $def, $seq ]
38 :     #
39 :     # \%representing
40 :     # Reference to a hash in which the keys are the ids of the
41 :     # representative sequences, for which the corresponding value
42 :     # is a list of ids of other sequences that are represented by
43 :     # that representive.
44 :     #
45 :     #
46 :     # Arguments (only \@seqs is required):
47 :     #
48 :     # $ref A reference sequence as [ $id, $def, $seq ]. If present, the
49 :     # reference sequence defines a focal point for the analysis. A
50 :     # representative sequence from each lineage in its vicinity will
51 :     # be retained, even though they are more similar than max_sim to
52 :     # the reference, or to each other. The reference will always be
53 :     # included in the representative set. A limit is put on the
54 :     # similarity of lineages retained by the reference sequence with
55 :     # the max_ref_sim option (default = 0.99). The reference sequence
56 :     # should not be repeated in the set of other sequences.
57 :     #
58 :     # \@seqs Set of sequences to be pruned. If there is no reference
59 :     # sequence, the fist sequence in this list will be the starting
60 :     # point for the analysis and will be retained, but all sequences
61 :     # more similar than max_sim to it will be removed (in contrast to
62 :     # a reference sequence, which retains a representative of each
63 :     # lineage in its vicinity). Sequences that fail the E-value test
64 :     # relative to the reference (or the fist sequence if there is no
65 :     # reference) are dropped.
66 :     #
67 :     # $max_sim Sequences with a higher similarity than max_sim to a retained
68 :     # sequence will be deleted. The details of the behaviour is
69 :     # modified by other options. (default = 0.80)
70 :     #
71 :     # \%options Key=>Value pairs that modify the behaviour:
72 :     #
73 :     # logfile Filehandle for a logfile of the progress. As each
74 :     # representative sequence is identified, its id is written
75 :     # to the logfile, followed by a tab separated list of the
76 :     # ids that it represents. Autoflush is set for the logfile.
77 :     # If the value supplied is not a reference to a GLOB, then
78 :     # the log is sent to STDOUT (which is probably not what you
79 :     # want in most cases). The behavior is intended to aid in
80 :     # following prgress, and in recovery of interupted runs.
81 :     #
82 :     # max_ref_sim
83 :     # Maximum similarity of any sequence to the reference. If
84 :     # max_ref_sim is less than max_sim, it is silently reset to
85 :     # max_sim. (default = 0.99, because 1.0 can be annoying)
86 :     #
87 :     # max_e_val Maximum E-value for BLAST (probably moot, but will help
88 :     # with performance) (default = 0.01)
89 :     #
90 :     # sim_meas Measure similarity for inclusion or exclusion by
91 :     # 'identity_fraction' (default), 'positive_fraction', or
92 :     # 'score_per_position'
93 :     #
94 :     # save_exp When there is a reference sequence, lineages more similar
95 :     # than max_sim will be retained near the reference. The
96 :     # default goal is to save one member of each lineage. If
97 :     # the initial representative of the lineage is seq1, we
98 :     # pose the question, "Are there sufficiently deep divisions
99 :     # within the lineage to seq1 that it they might be viewed
100 :     # as independent? That is, might there be another sequence,
101 :     # seq2 that so different from seq1 that we might want to see
102 :     # it also?
103 :     #
104 :     # +---------------------- ref
105 :     # |
106 :     # ---+ +-------------------- seq1
107 :     # +-+
108 :     # +-------------------- seq2
109 :     #
110 :     # Without any special treatment, if the similarity of seq1
111 :     # to ref ( S(seq1,ref) ) is greater than max_sim, seq1 would
112 :     # be the sole representative of thelineage containing both
113 :     # seq1 and seq2, because the similarity of seq1 to seq2
114 :     # ( S(seq1,seq2) ) is greater than S(seq1,ref). This can
115 :     # be altered by the value of save_exp. In terms of
116 :     # similarity, seq2 will be discarded if:
117 :     #
118 :     # S(seq1,seq2) > S(seq1,ref) ** save_exp, and
119 :     # S(seq1,seq2) > S(seq2,ref) ** save_exp
120 :     #
121 :     # The default behavior described above occurs when save_exp
122 :     # is 1. If save_exp < 1, then greater similarities between
123 :     # seq1 and seq2 are allowed. Reasonable values of save_exp
124 :     # are roughly 0.7 to 1.0. (At save_exp = 0, any similarity
125 :     # would be allowed; yuck.)
126 :     #
127 :     # stable If true (not undef, '', or 0), then the representatives
128 :     # will be chosen from as early in the list as possible (this
129 :     # facilitates augmentation of an existing list).
130 :     #
131 :     #-------------------------------------------------------------------------------
132 :     #
133 :     # Diagram of the pruning behavior:
134 :     #
135 :     # 0.5 0.6 0.7 0.8 0.9 1.0 Similarity
136 :     # |---------|---------|---------|---------|---------|
137 :     # .
138 :     # . + A
139 :     # . +---+
140 :     # . | + B
141 :     # . +---+
142 :     # . | +---- C
143 :     # . +----------+
144 :     # . | +-------- D
145 :     # . |
146 :     # +-----------+ +----------------- E
147 :     # | . +-+
148 :     # | . +----------------- F
149 :     # +----------------+ .
150 :     # | | . +--------------------------- G
151 :     # | +---+
152 :     # | . | +--------------------- H
153 :     # --+ . +-----+
154 :     # | . +--------------------- I
155 :     # | .
156 :     # | +------------------------------- J
157 :     # +----------------+ .
158 :     # | . +--------------------------- K
159 :     # +---+
160 :     # . +--------------------------- L
161 :     # .
162 :     # |---------|---------|---------|---------|---------|
163 :     # 0.5 0.6 0.7 0.8 0.9 1.0 Similarity
164 :     #
165 :     # In the above tree and max_sim = 0.70 and max_ref_sim = 0.99:
166 :     #
167 :     # With no reference sequence, and A first in the list, the representative
168 :     # sequences will be A, G, J and K.
169 :     #
170 :     # With A as the reference sequence and save_exp left at its default, the
171 :     # representative sequences will be A, C, D, E, G, J and K. B is excluded
172 :     # because it is more similar than max_ref_sim to A.
173 :     #
174 :     # With A as the reference sequence and save_exp = 0.8, the representative
175 :     # sequences will be A, C, D, E, F (comparably similar to A and E), G,
176 :     # H (comparably similar to A and G), J and K. The sequence L will be
177 :     # represented by K because L is much closer to K than to A.
178 :     #
179 :     # This oversimplifies the choice of representative of a cluster of related
180 :     # sequences. For example, whether G, H or I would represent the group of
181 :     # three actually depends on relative clock speed (slower is better) and
182 :     # sequence coverage (more complete is better). The actual order is by BLAST
183 :     # bit score (possibly combining independent segments).
184 :     #
185 :     # In addition, this discussion is in terms of a tree, but the calculations
186 :     # are based on a (partial) set of pairwise sequence similarities. Thus, the
187 :     # precise behavior is hard to predict, but should be similar to that described
188 :     # above.
189 :     #
190 :     #-------------------------------------------------------------------------------
191 :     #
192 :     # To construct a representative set of sequences relative to a reference
193 :     # sequence:
194 :     #
195 :     # 1. Prioritize sequences for keeping, from highest to lowest scoring
196 :     # relative to reference, as measured by blast score (bits).
197 :     # When stable is set, priority is from first to last in input file
198 :     # (a reference sequence should not be supplied).
199 :     #
200 :     # 2. Based on the similarity of each sequence to the reference and save_exp,
201 :     # compute sequence-specific values of max_sim:
202 :     #
203 :     # max_sim( seq_i ) = exp( save_exp * ln( seq_i_ref_sim ) )
204 :     #
205 :     # 3. Examine the next prioritized sequence (seq1).
206 :     #
207 :     # 4. If seq1 has been vetoed, go to 7.
208 :     #
209 :     # 5. Mark seq1 to keep.
210 :     #
211 :     # 6. Use blast to find similarities of seq1 to other sequences.
212 :     #
213 :     # 7. For each similar sequence (seq2):
214 :     #
215 :     # 7a. Skip if seq2 is marked to keep, or marked for veto
216 :     #
217 :     # 7b. Compute the maximum simiarity of seq1 and seq2 for retaining seq2:
218 :     #
219 :     # max_sim_1_2 = max( max_sim, max_sim( seq1 ), max_sim( seq2 ) )
220 :     #
221 :     # 7c. If similarity of seq1 and seq2 > max_sim, veto seq2
222 :     #
223 :     # 7d. Next seq2
224 :     #
225 :     # 8. If there are more sequences to examine, go to 3.
226 :     #
227 :     # 9. Collect the sequences marked for keeping.
228 :     #
229 :     #===============================================================================
230 :    
231 :     sub representative_sequences {
232 :     my $seqs = ( shift @_ || shift @_ ); # If $ref is undef, shift again
233 :     ref( $seqs ) eq "ARRAY"
234 :     or die "representative_sequences called with bad first argument\n";
235 :    
236 :     my ( $ref, $use_ref );
237 :     if ( ! ref( $seqs->[0] ) ) # First item was sequence entry, not list of entries
238 :     {
239 :     $ref = $seqs;
240 :     $seqs = shift @_;
241 :     ref( $seqs ) eq "ARRAY"
242 :     and ref( $seqs->[0] ) eq "ARRAY"
243 :     or die "representative_sequences called with bad sequences list\n";
244 :     $use_ref = 1;
245 :     }
246 :     else # First item was list of entries, split off first
247 :     {
248 :     ref( $seqs->[0] ) eq "ARRAY"
249 :     or die "representative_sequences called with bad sequences list\n";
250 :     $ref = shift @$seqs;
251 :     $use_ref = 0;
252 :     }
253 :    
254 :     my $max_sim = shift @_;
255 :     my $options;
256 :    
257 :     # Undocumented feature: skip max_sim (D = 0.8)
258 :    
259 :     if ( ref( $max_sim ) eq "HASH" )
260 :     {
261 :     $options = $max_sim;
262 :     $max_sim = undef;
263 :     }
264 :    
265 :     # If the above did not give us options, get them now:
266 :    
267 :     $options ||= ( shift @_ ) || {};
268 :    
269 :     # ---------------------------------------# Default values for options
270 :    
271 :     $max_sim ||= 0.80; # Retain 80% identity of less
272 :     my $logfile = undef; # Log file of sequences processed
273 :     my $max_ref_sim = 0.99; # Get rid of identical sequences
274 :     my $max_e_val = 0.01; # Blast E-value to decrease output
275 :     my $sim_meas = 'identity_fraction'; # Use sequence identity as measure
276 :     my $save_exp = 1.0; # Don't retain near equivalents
277 :     my $stable = 0; # Pick reps input order
278 :    
279 :     # Two questionable decisions:
280 :     # 1. Be painfully flexible on option names.
281 :     # 2. Silently fix bad parameter values.
282 :    
283 :     foreach ( keys %$options )
284 :     {
285 :     my $value = $options->{ $_ };
286 :     if ( m/^log/i ) # logfile
287 :     {
288 :     next if ! $value;
289 :     $logfile = ( ref $value eq "GLOB" ? $value : \*STDOUT );
290 :     select( ( select( $logfile ), $| = 1 )[0] ); # autoflush on
291 :     }
292 :     elsif ( m/ref/i ) # max_ref_sim
293 :     {
294 :     $value += 0;
295 :     $value = 0 if $value < 0;
296 :     $value = 1 if $value > 1;
297 :     $max_ref_sim = $value;
298 :     }
299 :     elsif ( m/max/i && m/sim/i ) # max(imum)_sim(ilarity)
300 :     {
301 :     $value += 0;
302 :     $value = 0 if $value < 0;
303 :     $value = 1 if $value > 1;
304 :     $max_sim = $value;
305 :     }
306 :     elsif ( m/max/i || m/[ep]_?val/i ) # Other "max" tests must come first
307 :     {
308 :     $value += 0;
309 :     $value = 0 if $value < 0;
310 :     $max_e_val = $value;
311 :     }
312 :     elsif ( m/sim/i || m/meas/i ) # sim(ilarity)_meas(ure)
313 :     {
314 :     $sim_meas = standardize_similarity_measure( $value );
315 :     }
316 :     elsif ( m/sav/i || m/exp/i ) # save_exp(onent)
317 :     {
318 :     $value += 0;
319 :     $value = 0 if $value < 0;
320 :     $value = 1 if $value > 1;
321 :     $save_exp = $value;
322 :     }
323 :     elsif ( m/stab/i ) # stable order
324 :     {
325 :     $stable = $value ? 1 : 0;
326 :     }
327 :     else
328 :     {
329 :     print STDERR "WARNING: representative_sequences bad option ignored: '$_' => '$value'\n";
330 :     }
331 :     }
332 :    
333 :     # Silent sanity check. This should not happen, as it is almost equivalent
334 :     # to making no reference sequence.
335 :    
336 :     $max_ref_sim = $max_sim if ( $max_ref_sim < $max_sim );
337 :    
338 :     # Do the analysis
339 :    
340 :     my $ref_id = $ref->[0];
341 :     my $ref_seq = $ref->[2];
342 :    
343 :     # Build a list of the ids (without ref) and an index for the sequence entries:
344 :    
345 :     my @seq_id = map { $_->[0] } @$seqs;
346 :     my $seq_ind = { map { @{$_}[0] => $_ } ( $ref, @$seqs ) };
347 :    
348 :     # Make a lookup table of the sequence number, for use in reording
349 :     # sequences later:
350 :    
351 :     my $n = 0;
352 :     my %ord = ( map { @$_[0] => ++$n } @$seqs );
353 :    
354 :     # Build blast database (it includes the reference):
355 :    
356 :     my $protein = are_protein( $seqs );
357 :     my $db = ( $tmp_dir ? "$tmp_dir/" : "" ) . "tmp_blast_db_$$";
358 :     make_blast_db( $db, [ $ref, @$seqs ], $protein );
359 :    
360 :     # Search query against new database
361 :    
362 :     my $max = 3 * @$seqs; # Alignments to keep
363 :     my $self = 1; # Keep self match (for its bit score)
364 :    
365 :     my $blast_opt = "-e $max_e_val -v $max -b $max -F F -a 2";
366 :    
367 :     # Do the blast analysis. Returned records are of the form:
368 :     #
369 :     # 0 1 2 3 4 5 6 7 8 9 10 11
370 :     # [ qid, qdef, qlen, sid, sdef, slen, scr, e_val, n_mat, n_id, n_pos, n_gap ]
371 :    
372 :     my $prog = $protein ? 'blastp' : 'blastn';
373 :     $blast_opt .= " -r 1 -q -1" if ! $protein;
374 :     my @ref_hits = top_blast_per_subject( $prog, $db, $ref_id, $ref_seq, $self, $blast_opt );
375 :    
376 :     # First hit is always a perfect match, so we get bits per position:
377 :     # This is only used if the measure is bits per position
378 :    
379 :     my $ref_bpp = $ref_hits[0]->[6] / $ref_hits[0]->[8];
380 :    
381 :     # Remove self match (might not be first if there are identical sequences):
382 :    
383 :     my %hit = ();
384 :     @ref_hits = grep { my $sid = $_->[3]; $hit{ $sid } = 1; ( $sid ne $ref_id ) } @ref_hits;
385 :    
386 :     my %keep = ();
387 :     $keep{ $ref_id } = [];
388 :     my %veto = ();
389 :     my $n_to_do = @ref_hits;
390 :     my $rebuild_d_n = 40;
391 :     my $last_rebuild = 1.5 * $rebuild_d_n;
392 :     my $rebuild = ( $n_to_do > $last_rebuild ) ? $n_to_do - $rebuild_d_n : 0;
393 :    
394 :     # Sequence-specific maximum similarities:
395 :    
396 :     my %max_sim = map { ( $_ => $max_sim ) } @seq_id;
397 :    
398 :     foreach ( @ref_hits )
399 :     {
400 :     my $id = $_->[3];
401 :     my $sim = seq_similarity( $_, $sim_meas, $ref_bpp );
402 :    
403 :     if ( $sim > ( $use_ref ? $max_ref_sim : $max_sim ) )
404 :     {
405 :     $veto{ $id } = 1;
406 :     push @{ $keep{ $ref_id } }, $id; # Log the sequences represented
407 :     $n_to_do--;
408 :     }
409 :     else
410 :     {
411 :     my $max_sim_i = exp( $save_exp * log( $sim ) );
412 :     $max_sim{ $id } = $max_sim_i if ( $max_sim_i > $max_sim );
413 :     }
414 :     }
415 :    
416 :    
417 :     if ( $logfile )
418 :     {
419 :     print $logfile join( "\t", $ref_id, @{ $keep{ $ref_id } } ), "\n";
420 :     }
421 :    
422 :     # Search each sequence against the database.
423 :     # If the order is to be stable, reorder hits to match input order.
424 :    
425 :     my ( $id1, $seq1, $max_sim_1, $id2, $max_sim_2, $bpp_max );
426 :     my @ids_to_do = map { $_->[3] } @ref_hits;
427 :     @ids_to_do = sort { $ord{ $a } <=> $ord{ $b } } @ids_to_do if $stable;
428 :    
429 :     while ( $id1 = shift @ids_to_do )
430 :     {
431 :     next if $veto{ $id1 };
432 :    
433 :     # Is it time to rebuild a smaller BLAST database? This helps
434 :     # significantly in the overall performance.
435 :    
436 :     if ( $n_to_do <= $rebuild )
437 :     {
438 :     if ( $protein ) { unlink $db, "$db.psq", "$db.pin", "$db.phr" }
439 :     else { unlink $db, "$db.nsq", "$db.nin", "$db.nhr" }
440 :     make_blast_db( $db, [ map { $seq_ind->{ $_ } } # id to sequence entry
441 :     grep { ! $veto{ $_ } } # id not vetoed
442 :     ( $id1, @ids_to_do ) # remaining ids
443 :     ],
444 :     $protein
445 :     );
446 :     $rebuild = ( $n_to_do > $last_rebuild ) ? $n_to_do - $rebuild_d_n : 0;
447 :     }
448 :    
449 :     $n_to_do--;
450 :     $keep{ $id1 } = [];
451 :    
452 :     $max_sim_1 = $max_sim{ $id1 };
453 :     $bpp_max = undef;
454 :     $seq1 = $seq_ind->{$id1}->[2];
455 :     foreach ( top_blast_per_subject( $prog, $db, $id1, $seq1, $self, $blast_opt ) )
456 :     {
457 :     $bpp_max ||= $_->[6] / $_->[8];
458 :     $id2 = $_->[3];
459 :     next if ( $veto{ $id2 } || $keep{ $id2 } );
460 :     $max_sim_2 = $max_sim{ $id2 };
461 :     $max_sim_2 = $max_sim_1 if ( $max_sim_1 > $max_sim_2 );
462 :     if ( seq_similarity( $_, $sim_meas, $bpp_max ) > $max_sim_2 )
463 :     {
464 :     $veto{ $id2 } = 1;
465 :     push @{ $keep{ $id1 } }, $id2; # Log the sequences represented
466 :     $n_to_do--;
467 :     }
468 :     }
469 :    
470 :     if ( $logfile )
471 :     {
472 :     print $logfile join( "\t", $id1, @{ $keep{ $id1 } } ), "\n";
473 :     }
474 :     }
475 :    
476 :     if ( $protein ) { unlink $db, "$db.psq", "$db.pin", "$db.phr" }
477 :     else { unlink $db, "$db.nsq", "$db.nin", "$db.nhr" }
478 :    
479 :     # Return the surviving sequence entries, and optionally the hash of
480 :     # ids represented by each survivor:
481 :    
482 :     my $kept = [ $ref, grep { $keep{ $_->[0] } } @$seqs ];
483 :    
484 :     wantarray ? ( $kept, \%keep, [ grep { ! $hit{ $_->[0] } } @$seqs ] ) : $kept;
485 :     }
486 :    
487 :    
488 :     #===============================================================================
489 :     # Build or add to a set of representative sequences.
490 :     #
491 :     # \@reps = rep_seq_2( \@reps, \@new, \%options );
492 :     # \@reps = rep_seq_2( \@new, \%options );
493 :     #
494 :     # or
495 :     #
496 :     # ( \@reps, \%representing ) = rep_seq_2( \@reps, \@new, \%options );
497 :     # ( \@reps, \%representing ) = rep_seq_2( \@new, \%options );
498 :     #
499 :     #===============================================================================
500 :    
501 :     sub rep_seq_2
502 :     {
503 :     ref $_[0] eq 'ARRAY'
504 :     or print STDERR "First parameter of rep_seq_2 must be an ARRAY reference\n"
505 :     and return undef;
506 :    
507 :     my ( $reps, $seqs, $options ) = ( [], undef, {} );
508 :    
509 :     if ( @_ == 1 )
510 :     {
511 :     $seqs = shift;
512 :     }
513 :    
514 :     elsif ( @_ == 2 )
515 :     {
516 :     if ( ref $_[1] eq 'ARRAY' ) { ( $reps, $seqs ) = @_ }
517 :     elsif ( ref $_[1] eq 'HASH' ) { ( $seqs, $options ) = @_ }
518 :     else
519 :     {
520 :     print STDERR "Second parameter of rep_seq_2 must be an ARRAY or HASH reference\n";
521 :     return undef;
522 :     }
523 :     }
524 :    
525 :     elsif ( @_ == 3 )
526 :     {
527 :     if ( ref $_[1] ne 'ARRAY' )
528 :     {
529 :     print STDERR "Second parameter of 3 parameter rep_seq_2 must be an ARRAY reference\n";
530 :     return undef;
531 :     }
532 :     if ( ref $_[2] ne 'HASH' )
533 :     {
534 :     print STDERR "Third parameter of 3 parameter rep_seq_2 must be a HASH reference\n";
535 :     return undef;
536 :     }
537 :    
538 :     ( $reps, $seqs, $options ) = @_;
539 :     }
540 :     else
541 :     {
542 :     print STDERR "rep_seq_2 called with @{[scalar @_]} parameters\n";
543 :     return undef;
544 :     }
545 :    
546 :     # ---------------------------------------# Default values for options
547 :    
548 :     my $max_sim = 0.80; # Retain 80% identity of less
549 :     my $logfile = undef; # Log file of sequences processed
550 :     my $max_e_val = 0.01; # Blast E-value to decrease output
551 :     my $sim_meas = 'identity_fraction'; # Use sequence identity as measure
552 :    
553 :     # Two questionable decisions:
554 :     # 1. Be painfully flexible on option names.
555 :     # 2. Silently fix bad parameter values.
556 :    
557 :     foreach ( keys %$options )
558 :     {
559 :     my $value = $options->{ $_ };
560 :     if ( m/^log/i ) # logfile
561 :     {
562 :     next if ! $value;
563 :     $logfile = ( ref $value eq "GLOB" ? $value : \*STDOUT );
564 :     select( ( select( $logfile ), $| = 1 )[0] ); # autoflush on
565 :     }
566 :     elsif ( m/max/i && m/sim/i ) # max(imum)_sim(ilarity)
567 :     {
568 :     $value += 0;
569 :     $value = 0 if $value < 0;
570 :     $value = 1 if $value > 1;
571 :     $max_sim = $value;
572 :     }
573 :     elsif ( m/max/i || m/[ep]_?val/i ) # Other "max" tests must come first
574 :     {
575 :     $value += 0;
576 :     $value = 0 if $value < 0;
577 :     $max_e_val = $value;
578 :     }
579 :     elsif ( m/sim/i || m/meas/i ) # sim(ilarity)_meas(ure)
580 :     {
581 :     $sim_meas = standardize_similarity_measure( $value );
582 :     }
583 :     else
584 :     {
585 :     print STDERR "WARNING: rep_seq_2 bad option ignored: '$_' => '$value'\n";
586 :     }
587 :     }
588 :    
589 :     # Check sequence ids for duplicates:
590 :    
591 :     my $reps2 = [];
592 :     my $seen = {};
593 :    
594 :     my $id;
595 :     foreach ( @$reps )
596 :     {
597 :     $id = $_->[0];
598 :     if ( $seen->{ $id }++ )
599 :     {
600 :     print STDERR "Duplicate sequence id '$id' skipped by rep_seq_2\n";
601 :     }
602 :     else
603 :     {
604 :     push @$reps2, $_;
605 :     }
606 :     }
607 :    
608 :     my $seqs2 = [];
609 :     foreach ( @$seqs )
610 :     {
611 :     $id = $_->[0];
612 :     if ( $seen->{ $id }++ )
613 :     {
614 :     print STDERR "Duplicate sequence id '$_[0]' skipped by rep_seq_2\n";
615 :     }
616 :     else
617 :     {
618 :     push @$seqs2, $_;
619 :     }
620 :     }
621 :    
622 :     # If no preexisting representatives, then take first sequence:
623 :    
624 :     @$reps2 or @$reps2 = ( shift @$seqs2 );
625 :    
626 :     if ( $logfile ) { foreach ( @$reps2 ) { print $logfile "$_->[0]\n" } }
627 :    
628 :     # Search each rep sequence against itself for max_bpp
629 :    
630 :     my $db = ( $tmp_dir ? "$tmp_dir/" : "" ) . "tmp_blast_db_$$";
631 :     my $protein = are_protein( $reps2 );
632 :     my %max_bpp;
633 :     my $entry;
634 :     foreach $entry ( @$reps2 )
635 :     {
636 :     $max_bpp{ $entry->[0] } = self_bpp( $db, $entry, $protein );
637 :     }
638 :    
639 :     my $naln = 10; # Alignments to make
640 :     my $self = 1; # Self match will never occur
641 :     my $prog = $protein ? 'blastp' : 'blastn';
642 :     my $blast_opt = "-e $max_e_val -v $naln -b $naln -F F -a 2";
643 :     $blast_opt .= " -r 1 -q -1" if ! $protein;
644 :    
645 :     # List of who is represented by a sequence:
646 :    
647 :     my %keep = map { $_->[0] => [] } @$reps2;
648 :    
649 :     # Search each sequence against the database.
650 :    
651 :     my ( $seq, $bpp_max );
652 :     my $newdb = 1;
653 :    
654 :     foreach $entry ( @$seqs2 )
655 :     {
656 :     # Is it time to rebuild a BLAST database?
657 :    
658 :     if ( $newdb )
659 :     {
660 :     make_blast_db( $db, $reps2, $protein );
661 :     $newdb = 0;
662 :     }
663 :    
664 :     # Do the blast analysis. Returned records are of the form:
665 :     #
666 :     # 0 1 2 3 4 5 6 7 8 9 10 11
667 :     # [ qid, qdef, qlen, sid, sdef, slen, scr, e_val, n_mat, n_id, n_pos, n_gap ]
668 :    
669 :     $id = $entry->[0];
670 :     $seq = $entry->[2];
671 :     my ( $tophit ) = sort { $b->[0] <=> $a->[0] }
672 :     grep { $_->[0] >= $max_sim }
673 :     map { [ seq_similarity( $_, $sim_meas, $max_bpp{ $_->[3] } ), $_ ] }
674 :     top_blast_per_subject( $prog, $db, $id, $seq, $self, $blast_opt );
675 :    
676 :    
677 :     # It matches an existing representative
678 :    
679 :     if ( $tophit )
680 :     {
681 :     # print STDERR join(", ", $tophit->[0], @{$tophit->[1]} ), "\n";
682 :     push @{ $keep{ $tophit->[1]->[3] } }, $id;
683 :     print $logfile "$id\t$tophit->[1]->[3]\n" if $logfile;
684 :     }
685 :    
686 :     # It is a new representative
687 :    
688 :     else
689 :     {
690 :     push @$reps2, $entry;
691 :     $keep{ $id } = [];
692 :     $max_bpp{ $id } = self_bpp( $db, $entry, $protein );
693 :     $newdb = 1;
694 :     print $logfile "$id\n" if $logfile;
695 :     }
696 :     }
697 :    
698 :     if ( $protein ) { unlink $db, "$db.psq", "$db.pin", "$db.phr" }
699 :     else { unlink $db, "$db.nsq", "$db.nin", "$db.nhr" }
700 :    
701 :     # Return the surviving sequence entries, and optionally the hash of
702 :     # ids represented by each survivor:
703 :    
704 :     wantarray ? ( $reps2, \%keep ) : $reps2;
705 :     }
706 :    
707 :    
708 :     #===============================================================================
709 :     # Try to figure out the sequence similarity measure that is being requested:
710 :     #
711 :     # $type = standardize_similarity_measure( $requested_type )
712 :     #
713 :     #===============================================================================
714 :    
715 :     sub standardize_similarity_measure
716 :     { my ( $req_meas ) = @_;
717 :     return ( ! $req_meas ) ? 'identity_fraction'
718 :     : ( $req_meas =~ /id/i ) ? 'identity_fraction'
719 :     : ( $req_meas =~ /sc/i ) ? 'score_per_position'
720 :     : ( $req_meas =~ /spp/i ) ? 'score_per_position'
721 :     : ( $req_meas =~ /bit/i ) ? 'score_per_position'
722 :     : ( $req_meas =~ /bpp/i ) ? 'score_per_position'
723 :     : ( $req_meas =~ /tiv/i ) ? 'positive_fraction'
724 :     : ( $req_meas =~ /pos_/i ) ? 'positive_fraction'
725 :     : ( $req_meas =~ /ppp/i ) ? 'positive_fraction'
726 :     : 'identity_fraction';
727 :     }
728 :    
729 :    
730 :     #===============================================================================
731 :     # Caluculate sequence similarity according to the requested measure:
732 :     #
733 :     # $similarity = seq_similarity( $hit, $measure, $bpp_max )
734 :     #
735 :     # $hit is a structure with blast information:
736 :     #
737 :     # [ qid, qdef, qlen, sid, sdef, slen, scr, e_val, n_mat, n_id, n_pos, n_gap ]
738 :     #===============================================================================
739 :    
740 :     sub seq_similarity
741 :     { my ( $hit, $measure, $bpp_max ) = @_;
742 :     return ( @$hit < 11 ) ? undef
743 :     : ( $measure =~ /^sc/ ) ? $hit->[ 6] / ( $hit->[8] * ( $bpp_max || 2 ) )
744 :     : ( $measure =~ /^po/ ) ? $hit->[10] / $hit->[8]
745 :     : $hit->[ 9] / $hit->[8]
746 :     }
747 :    
748 :    
749 :     #===============================================================================
750 :     # Caluculate self similarity of a sequence in bits per position:
751 :     #
752 :     # $max_bpp = self_bpp( $db_name, $entry, $protein )
753 :     #
754 :     #===============================================================================
755 :    
756 :     sub self_bpp
757 :     {
758 :     my ( $db, $entry, $protein ) = @_;
759 :    
760 :     # Build blast database:
761 :    
762 :     make_blast_db( $db, [ $entry ], $protein );
763 :    
764 :     # Search sequence against the database
765 :    
766 :     my $id = $entry->[0];
767 :     my $seq = $entry->[2];
768 :     my $self = 1; # Self match will never occur
769 :    
770 :     my $prog = $protein ? 'blastp' : 'blastn';
771 :     my $blast_opt = "-v 1 -b 1 -F F -a 2";
772 :     $blast_opt .= " -r 1 -q -1" if ! $protein;
773 :    
774 :     # Do the blast analysis. Returned records are of the form:
775 :     #
776 :     # 0 1 2 3 4 5 6 7 8 9 10 11
777 :     # [ qid, qdef, qlen, sid, sdef, slen, scr, e_val, n_mat, n_id, n_pos, n_gap ]
778 :    
779 :     my ( $hit ) = top_blast_per_subject( $prog, $db, $id, $seq, $self, $blast_opt );
780 :     # print STDERR join( ", ", @$hit ), "\n";
781 :    
782 :     # First hit is always a perfect match, so we get bits per position:
783 :     # This is only used if the measure is bits per position
784 :    
785 :     $hit->[6] / $hit->[8];
786 :     }
787 :    
788 :    
789 :     #===============================================================================
790 :     # Make a blast databse from a set of sequence entries. The type of database
791 :     # (protein or nucleic acid) is quessed from the sequence data.
792 :     #
793 :     # make_blast_db( $db_filename, \@seq_entries, $protein )
794 :     #
795 :     # Sequence entries have the form: [ $id, $def, $seq ]
796 :     #===============================================================================
797 :    
798 :     sub make_blast_db
799 :     { my ( $db, $seqs, $protein ) = @_;
800 :    
801 :     open FH, ">$db" or die "Could not create blast database file \"$db\"";
802 :     foreach ( @$seqs )
803 :     {
804 :     print FH ">$_->[0]", ( $_->[1] ? " $_->[1]" : () ), "\n$_->[2]\n";
805 :     }
806 :     close FH;
807 :    
808 :     my $is_prot = $protein ? 'T' : 'F';
809 :     system "$formatdb -p $is_prot -i '$db'";
810 :     }
811 :    
812 :    
813 :     #===============================================================================
814 :     # The type of data (protein or nucleic acid) is quessed from the sequences.
815 :     #
816 :     # are_protein( \@seq_entries )
817 :     #
818 :     # Sequence entries have the form: [ $id, $def, $seq ]
819 :     #===============================================================================
820 :    
821 :     sub are_protein
822 :     { my ( $seqs ) = @_;
823 :     my ( $nt, $aa ) = ( 0, 0 );
824 :     foreach ( @$seqs )
825 :     {
826 :     my $s = $_->[2];
827 :     $nt += $s =~ tr/ACGTacgt//d;
828 :     $aa += $s =~ tr/A-Za-z//d;
829 :     }
830 :     ( $nt < 3 * $aa ) ? 1 : 0;
831 :     }
832 :    
833 :    
834 :     #===============================================================================
835 :     # Blast a subject against a datbase, saving only top hit per subject
836 :     #
837 :     # Return:
838 :     #
839 :     # [ qid, qdef, qlen, sid, sdef, slen, scr, e_val, n_mat, n_id, n_pos, n_gap ]
840 :     #
841 :     #===============================================================================
842 :    
843 :     # Use a bare block to compartmentalize a counter:
844 :    
845 :     { my $cnt = 0;
846 :    
847 :     sub top_blast_per_subject
848 :     { my ( $prog, $db, $qid, $qseq, $self, $blast_opt, $sort, $no_merge ) = @_;
849 :    
850 :     $cnt++;
851 :     my $query_file = ( $tmp_dir ? "$tmp_dir/" : "" ) . "tmp_blast_seq_${$}_$cnt";
852 :     my $QFILE;
853 :     open $QFILE, ">$query_file" or die "Could not create sequence file \"$query_file\"";
854 :     print $QFILE ">$qid\n$qseq\n";
855 :     close $QFILE;
856 :    
857 :     my $blast_cmd = "$blastall -p $prog -d '$db' -i '$query_file' $blast_opt";
858 :    
859 :     my $BPIPE;
860 :     open $BPIPE, "$blast_cmd |" or die "Could open blast pipe\n";
861 :     my $sims = integrate_blast_segments( $BPIPE, $sort, $no_merge, $self );
862 :     close $BPIPE;
863 :     unlink $query_file;
864 :    
865 :     my $pq = ""; # Previous query id
866 :     my $ps = ""; # Previous subject id
867 :     my $keep;
868 :    
869 :     grep { $keep = ( $pq ne $_->[0] ) || ( $ps ne $_->[3] );
870 :     $pq = $_->[0];
871 :     $ps = $_->[3];
872 :     $keep && ( $self || ( $pq ne $ps ) );
873 :     } @$sims;
874 :     }
875 :     }
876 :    
877 :    
878 :     #===============================================================================
879 :     # Read output of rationalize blast and assemble minimally overlapping segments
880 :     # into a total score for each subject sequence. For each query, sort matches
881 :     # into user-chosen order (D = total score):
882 :     #
883 :     # @sims = integrate_blast_segments( \*FILEHANDLE, $sort_order, $no_merge )
884 :     # \@sims = integrate_blast_segments( \*FILEHANDLE, $sort_order, $no_merge )
885 :     #
886 :     # Allowed sort orders are 'score', 'score_per_position', 'identity_fraction',
887 :     # and 'positive_fraction' (matched very flexibly).
888 :     #
889 :     # Returned sims (e_val is only for best HSP, not any composite):
890 :     #
891 :     # [ qid, qdef, qlen, sid, sdef, slen, scr, e_val, n_mat, n_id, n_pos, n_gap ]
892 :     #
893 :     # There is a strategic decision to not read the blast output from memory;
894 :     # it could be enormous. This cuts the flexibility some.
895 :     #===============================================================================
896 :     #
897 :     # coverage fields:
898 :     #
899 :     # [ scr, e_val, n_mat, n_id, n_pos, n_gap, dir, [ intervals_covered ] ]
900 :     #
901 :     #===============================================================================
902 :    
903 :     sub integrate_blast_segments
904 :     { my ( $fh, $order, $no_merge, $self ) = @_;
905 :     $fh ||= \*STDIN;
906 :     ( ref( $fh ) eq "GLOB" ) || die "integrate_blast_segments called without a filehandle\n";
907 :    
908 :     $order = ( ! $order ) ? 'score'
909 :     : ( $order =~ /sc/i ) ? ( $order =~ /p/i ? 'score_per_position' : 'score' )
910 :     : ( $order =~ /bit/i ) ? ( $order =~ /p/i ? 'score_per_position' : 'score' )
911 :     : ( $order =~ /spp/i ) ? 'score_per_position'
912 :     : ( $order =~ /id/i ) ? 'identity_fraction'
913 :     : ( $order =~ /tiv/i ) ? 'positive_fraction'
914 :     : 'score';
915 :    
916 :     my $max_frac_overlap = 0.2;
917 :    
918 :     my ( $qid, $qdef, $qlen, $sid, $sdef, $slen );
919 :     my ( $scr, $e_val, $n_mat, $n_id, $n_pos, $n_gap );
920 :     my ( $ttl_scr, $ttl_mat, $ttl_id, $ttl_pos, $ttl_gap );
921 :     my @sims = ();
922 :     my @qsims = ();
923 :     my $coverage = undef;
924 :     my $record;
925 :    
926 :     while ( $_ = next_blast_record( $fh, $self ) )
927 :     {
928 :     chomp;
929 :     if ( $_->[0] eq 'Query=' )
930 :     {
931 :     if ( $coverage )
932 :     {
933 :     push @qsims, [ $sid, $sdef, $slen, @$coverage[ 0 .. 5 ] ];
934 :     $coverage = undef;
935 :     }
936 :     if ( @qsims ) { push @sims, order_query_sims( $qid, $qdef, $qlen, \@qsims, $order ) }
937 :     ( undef, $qid, $qdef, $qlen ) = @$_;
938 :     $sid = undef;
939 :     @qsims = ();
940 :     }
941 :     elsif ( $_->[0] eq '>' )
942 :     {
943 :     if ( $coverage )
944 :     {
945 :     push @qsims, [ $sid, $sdef, $slen, @$coverage[ 0 .. 5 ] ];
946 :     $coverage = undef;
947 :     }
948 :     next if ! $qid;
949 :     ( undef, $sid, $sdef, $slen ) = @$_;
950 :     }
951 :     elsif ( $_->[0] eq 'HSP' && $sid )
952 :     {
953 :     $coverage = integrate_HSP( $coverage, $_, $max_frac_overlap, $no_merge );
954 :     }
955 :     }
956 :    
957 :     if ( $coverage ) { push @qsims, [ $sid, $sdef, $slen, @$coverage[ 0 .. 5 ] ] }
958 :    
959 :     if ( @qsims ) { push @sims, order_query_sims( $qid, $qdef, $qlen, \@qsims, $order ) }
960 :    
961 :     wantarray ? @sims : \@sims;
962 :     }
963 :    
964 :    
965 :     #===============================================================================
966 :     #
967 :     # Try to integrate non-conflicting HSPs for the same subject sequence. The
968 :     # conflicts are only assessed from the standpoint of the query, at least for
969 :     # now. We could track the subject sequence coverage as well (to avoid a direct
970 :     # repeat in the query from matching the same subject twice).
971 :     #
972 :     # $new_coverage = integrate_HSP( $coverage, $hsp, $max_frac_overlap, $no_merge )
973 :     #
974 :     # 0 1 2 3 4 5 6 7
975 :     # $coverage = [ scr, e_val, n_mat, n_id, n_pos, n_gap, dir, [ intervals_covered ] ]
976 :     #
977 :     # $coverage should be undefined at the first call; the function intiallizes
978 :     # all of the fields from the first HSP. scr, n_mat, n_id, n_pos, and n_gap
979 :     # are sums over the combined HSPs. e_val is based only of the first HSP.
980 :     #
981 :     # 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
982 :     # $hsp = [ 'HSP', scr, e_val, n_seg, e_val2, n_mat, n_id, n_pos, n_gap, dir, s1, e1, sq1, s2, e2, sq2 ]
983 :     #
984 :     # $max_frac_overlap Amount of the new HSP that is allowed to overlap already
985 :     # incorporated HSPs
986 :     #
987 :     # $no_merge Disable the merging of multiple HSPs. The structure will
988 :     # be filled in from the first HSP and left unchanged though
989 :     # subsequence calls. This simplifies the program structure.
990 :     #
991 :     # Fitting a new HSP into covered intervals:
992 :     #
993 :     # 1 qlen
994 :     # |---------------------------------------------------------------| query
995 :     # ------------ --------------- covered
996 :     # ------------- new match
997 :     # l r
998 :     #
999 :     #===============================================================================
1000 :    
1001 :     sub integrate_HSP
1002 :     { my ( $coverage, $hsp, $max_frac_overlap, $no_merge ) = @_;
1003 :     my ( undef, $scr, $e_val, undef, undef, $n_mat, $n_id, $n_pos, $n_gap, $dir, $s1, $e1 ) = @$hsp;
1004 :    
1005 :     # Ignore frame; just use direction of match:
1006 :    
1007 :     $dir = substr( $dir, 0, 1 );
1008 :    
1009 :     # Orient by left and right ends:
1010 :    
1011 :     my ( $l, $r ) = ( $e1 > $s1 ) ? ( $s1, $e1 ) : ( $e1, $s1 );
1012 :    
1013 :     # First HSP for the subject sequence:
1014 :    
1015 :     if ( ! $coverage )
1016 :     {
1017 :     return [ $scr, $e_val, $n_mat, $n_id, $n_pos, $n_gap, $dir, [ [ $s1, $e1 ] ] ];
1018 :     }
1019 :    
1020 :     # Not first; must be same direction to combine (also test no_merge here):
1021 :    
1022 :     return $coverage if ( $no_merge || ( $dir ne $coverage->[6] ) );
1023 :    
1024 :     # Not first; must fall in a gap of query sequence coverage:
1025 :    
1026 :     my @intervals = @{ $coverage->[7] };
1027 :     my $max_overlap = $max_frac_overlap * ( $r - $l + 1 );
1028 :     my $prev_end = 0;
1029 :     my $next_beg = $intervals[0]->[0];
1030 :     my @used = ();
1031 :     while ( $next_beg <= $l ) # *** Sequential search could be made binary
1032 :     {
1033 :     $prev_end = $intervals[0]->[1];
1034 :     push @used, scalar shift @intervals;
1035 :     $next_beg = @intervals ? $intervals[0]->[0] : 1e10;
1036 :     }
1037 :    
1038 :     my $overlap = ( ( $l <= $prev_end ) ? ( $prev_end - $l + 1 ) : 0 )
1039 :     + ( ( $r >= $next_beg ) ? ( $r - $next_beg + 1 ) : 0 );
1040 :     return $coverage if ( $overlap > $max_overlap );
1041 :    
1042 :     # Okay, we have passed the overlap test. We need to integrate the
1043 :     # match into the coverage description. Yes, I know that this counts
1044 :     # the overlap region. We could pro rate it, but that is messy too:
1045 :    
1046 :     $coverage->[0] += $scr;
1047 :     $coverage->[2] += $n_mat;
1048 :     $coverage->[3] += $n_id;
1049 :     $coverage->[4] += $n_pos;
1050 :     $coverage->[5] += $n_gap;
1051 :    
1052 :     # Refigure the covered intervals, fusing intervals separated by a
1053 :     # gap of less than 10:
1054 :    
1055 :     my $min_gap = 10;
1056 :     if ( $l <= $prev_end + $min_gap )
1057 :     {
1058 :     if ( @used ) { $l = $used[-1]->[0]; pop @used }
1059 :     else { $l = 1 }
1060 :     }
1061 :     if ( $r >= $next_beg - $min_gap )
1062 :     {
1063 :     if ( @intervals ) { $r = $intervals[0]->[1]; shift @intervals }
1064 :     else { $r = 1e10 }
1065 :     }
1066 :    
1067 :     $coverage->[7] = [ @used, [ $l, $r ], @intervals ];
1068 :    
1069 :     return $coverage;
1070 :     }
1071 :    
1072 :    
1073 :     #===============================================================================
1074 :     # Sort the blast matches by the desired criterion:
1075 :     #
1076 :     # @sims = order_query_sims( $qid, $qdef, $qlen, \@qsims, $order )
1077 :     #
1078 :     # Allowed sort orders are 'score', 'score_per_position', 'identity_fraction',
1079 :     # and 'positive_fraction'
1080 :     #
1081 :     # @qsims fields:
1082 :     #
1083 :     # 0 1 2 3 4 5 6 7 8
1084 :     # [ sid, sdef, slen, scr, e_val, n_mat, n_id, n_pos, n_gap ]
1085 :     #
1086 :     #===============================================================================
1087 :    
1088 :     sub order_query_sims
1089 :     { my ( $qid, $qdef, $qlen, $qsims, $order ) = @_;
1090 :    
1091 :     my @sims;
1092 :     if ( $order eq 'score_per_position' )
1093 :     {
1094 :     @sims = map { [ $_->[5] ? $_->[3]/$_->[5] : 0, $_ ] } @$qsims;
1095 :     }
1096 :     elsif ( $order eq 'identity_fraction' )
1097 :     {
1098 :     @sims = map { [ $_->[5] ? $_->[6]/$_->[5] : 0, $_ ] } @$qsims;
1099 :     }
1100 :     elsif ( $order eq 'positive_fraction' )
1101 :     {
1102 :     @sims = map { [ $_->[5] ? $_->[7]/$_->[5] : 0, $_ ] } @$qsims;
1103 :     }
1104 :     else # Default is by 'score'
1105 :     {
1106 :     @sims = map { [ $_->[3], $_ ] } @$qsims;
1107 :     }
1108 :    
1109 :     map { [ $qid, $qdef, $qlen, @{$_->[1]} ] } sort { $b->[0] <=> $a->[0] } @sims;
1110 :     }
1111 :    
1112 :    
1113 :     1;

MCS Webmaster
ViewVC Help
Powered by ViewVC 1.0.3