/* $Id: pattern.c,v 1.24 2008/07/17 17:55:44 kazimird Exp $
* ===========================================================================
*
* PUBLIC DOMAIN NOTICE
* National Center for Biotechnology Information
*
* This software/database is a "United States Government Work" under the
* terms of the United States Copyright Act. It was written as part of
* the author's official duties as a United States Government employee and
* thus cannot be copyrighted. This software/database is freely available
* to the public for use. The National Library of Medicine and the U.S.
* Government have not placed any restriction on its use or reproduction.
*
* Although all reasonable efforts have been taken to ensure the accuracy
* and reliability of the software and data, the NLM and the U.S.
* Government do not and cannot warrant the performance or results that
* may be obtained by using this software or data. The NLM and the U.S.
* Government disclaim all warranties, express or implied, including
* warranties of performance, merchantability or fitness for any particular
* purpose.
*
* Please cite the author in any work or product based on this material.
*
* ===========================================================================
*
* Author: Ilya Dondoshansky
*
*/
/** @file pattern.c
* Functions for finding pattern matches in sequence.
* The following functions are defined here. @sa phi_lookup.h
*
*
* SPHIQueryInfoNew, SPHIQueryInfoFree, SPHIQueryInfoCopy - life cycle functions
* for the SPHIQueryInfo structure for saving pattern occurrences in query.
*
* Main API function to find and save pattern occurrences in query, and functions
* called from it:
*
* PHIGetPatternOccurrences
* FindPatternHits
* if ( pattern fits into a single word)
* s_FindHitsShortHead
* else if ( pattern fits into several words )
* s_FindHitsLong
* else if ( pattern contains parts longer than a word )
* s_FindHitsVeryLong
* calls s_FindHitsShortHead for every word and extends them
*
* For pattern occurrences in subject (database),
* FindPatternHits is called from PHIBlastScanSubject.
*
*
*/
#ifndef SKIP_DOXYGEN_PROCESSING
static char const rcsid[] =
"$Id: pattern.c,v 1.24 2008/07/17 17:55:44 kazimird Exp $";
#endif /* SKIP_DOXYGEN_PROCESSING */
#include
#include "pattern_priv.h"
void
_PHIGetRightOneBits(Int4 s, Int4 mask, Int4* rightOne, Int4* rightMaskOnly)
{
const int kCheckingMatches = s & mask; /*look for 1 bits in same position*/
Int4 right_index; /*loop index looking for 1 in kCheckingMatches*/
Int4 left_index; /*rightmost bit that is 1 in the mask only*/
left_index = -1;
for (right_index = 0; right_index < PHI_BITS_PACKED_PER_WORD; right_index++) {
if ((kCheckingMatches >> right_index) % 2 == 1)
break;
if ((mask >> right_index) %2 == 1)
left_index = right_index;
}
/* If there was no break from the loop, s and mask have no 1 bits in same
position, so rightOne should be 0. */
if (right_index == PHI_BITS_PACKED_PER_WORD)
right_index = 0;
*rightOne = right_index;
*rightMaskOnly = left_index;
}
/** Looks for 1 bits in the same position of s and mask
* Let R be the rightmost position where s and mask both have a 1.
* Let L < R be the rightmost position where mask has a 1, if any,
* or -1 otherwise.
* @param s Some number [in]
* @param mask Mask [in]
* @return (R - L).
*/
static Int4
s_LenOf(Int4 s, Int4 mask)
{
Int4 rightOne; /*loop index looking for 1 in kCheckingMatches*/
Int4 rightMaskOnly; /*rightmost bit that is 1 in the mask only*/
_PHIGetRightOneBits(s, mask, &rightOne, &rightMaskOnly);
return (rightOne - rightMaskOnly);
}
Int4
_PHIBlastFindHitsShort(Int4 *hitArray, const Uint1* seq, Int4 len1,
const SPHIPatternSearchBlk *pattern_blk)
{
Int4 i; /*loop index on sequence*/
Int4 prefixMatchedBitPattern = 0; /*indicates where pattern aligns
with seq; e.g., if value is 9 = 0101 then
last 3 chars of seq match first 3 positions in pattern
and last 1 char of seq matches 1 position of pattern*/
Int4 numMatches = 0; /*number of matches found*/
Int4 mask; /*mask of input pattern positions after which
a match can be declared*/
Int4 maskShiftPlus1; /*mask shifted left 1 plus 1 */
const SShortPatternItems* pattern_items = pattern_blk->one_word_items;
mask = pattern_items->match_mask;
maskShiftPlus1 = (mask << 1) + 1;
for (i = 0; i < len1; i++) {
/*shift the positions matched by 1 and try to match up against
the next character, also allow next character to match the
first position*/
prefixMatchedBitPattern =
((prefixMatchedBitPattern << 1) | maskShiftPlus1) &
pattern_items->whichPositionPtr[seq[i]];
if (prefixMatchedBitPattern & mask) {
/*first part of pair is index of place in seq where match
ends; second part is where match starts*/
hitArray[numMatches++] = i;
hitArray[numMatches++] = i - s_LenOf(prefixMatchedBitPattern, mask)+1;
if (numMatches == PHI_MAX_HIT)
{
/** @todo FIXME: Pass back an error message saying that
numMatches matches are saved, others discarded. */
break;
}
}
}
return numMatches;
}
/** Find hits when sequence is DNA and pattern is short returns twice the number
* of hits.
* @param hitArray Array of hits to pass back [out]
* @param seq The input sequence [in]
* @param pos Starting position [in]
* @param len Length of sequence seq [in]
* @param pattern_blk Carries variables that keep track of search
* parameters. [in]
* @return Number of hits found.
*/
static Int4
s_FindHitsShortDNA(Int4* hitArray, const Uint1* seq, Int4 pos, Int4 len,
const SPHIPatternSearchBlk *pattern_blk)
{
Uint4 prefixMatchedBitPattern; /*indicates where pattern aligns
with sequence*/
Uint4 tmp; /*intermediate result of masked comparisons*/
Int4 i; /*index on seq*/
Int4 end; /*count of number of 4-mer iterations needed*/
Int4 remain; /*0,1,2,3 DNA letters left over*/
Int4 j; /*index on suffixRemnant*/
Int4 twiceNumHits = 0; /*twice the number of hits*/
const SShortPatternItems* pattern_items = pattern_blk->one_word_items;
const Int4 kMatchMask = pattern_items->match_mask;
/* Mask to match agaist */
const Uint4 kMask2 = kMatchMask*PHI_BITS_PACKED_PER_WORD+15;
const Int4 kMaskShiftPlus1 = (kMatchMask << 1)+1; /* kMask2 shifted plus 1*/
if (pos != 0) {
pos = 4 - pos;
prefixMatchedBitPattern =
((kMatchMask * ((1 << (pos+1))-1)*2) + (1 << (pos+1))-1) &
pattern_items->dna_items->DNAwhichSuffixPosPtr[seq[0]];
seq++;
end = (len-pos)/4;
remain = (len-pos) % 4;
}
else {
prefixMatchedBitPattern = kMaskShiftPlus1;
end = len/4;
remain = len % 4;
}
for (i = 0; i < end; i++) {
if ( (tmp = (prefixMatchedBitPattern &
pattern_items->dna_items->DNAwhichPrefixPosPtr[seq[i]]))) {
for (j = 0; j < 4; j++) {
if (tmp & kMatchMask) {
hitArray[twiceNumHits++] = i*4 + j + pos;
hitArray[twiceNumHits++] = i*4 + j + pos -
s_LenOf(tmp & kMatchMask, kMatchMask) + 1;
}
tmp = (tmp << 1);
}
}
prefixMatchedBitPattern =
(((prefixMatchedBitPattern << 4) | kMask2) &
pattern_items->dna_items->DNAwhichSuffixPosPtr[seq[i]]);
}
/* In the last byte check bits only up to 'remain' */
if ( (tmp = (prefixMatchedBitPattern &
pattern_items->dna_items->DNAwhichPrefixPosPtr[seq[i]]))) {
for (j = 0; j < remain; j++) {
if (tmp & kMatchMask) {
hitArray[twiceNumHits++] = i*4+j + pos;
hitArray[twiceNumHits++] = i*4+j + pos -
s_LenOf(tmp & kMatchMask, kMatchMask) + 1;
}
tmp = (tmp << 1);
}
}
return twiceNumHits;
}
/** Top level routine to find hits when pattern has a short description.
* @param hitArray Array of hits to pass back [out]
* @param seq Input sequence [in]
* @param start Position to start at in seq [in]
* @param len Length of seq [in]
* @param is_dna 1 if and only if seq is a DNA sequence [in]
* @param pattern_blk Carries variables that keep track of search
* parameters. [in]
* @return Number of matches found.
*/
static Int4
s_FindHitsShortHead(Int4* hitArray, const Uint1* seq, Int4 start, Int4 len,
Uint1 is_dna, const SPHIPatternSearchBlk *pattern_blk)
{
if (is_dna)
return s_FindHitsShortDNA(hitArray, &seq[start/4], start % 4, len, pattern_blk);
return _PHIBlastFindHitsShort(hitArray, &seq[start], len, pattern_blk);
}
void
_PHIPatternWordsLeftShift(Int4 *a, Uint1 b, Int4 numWords)
{
Int4 x;
Int4 i; /*index on words*/
/* Overflow threshold */
const Int4 kOverflowThreshold = (1 << PHI_BITS_PACKED_PER_WORD);
for (i = 0; i < numWords; i++) {
x = (a[i] << 1) + b;
if (x >= kOverflowThreshold) {
a[i] = x - kOverflowThreshold;
b = 1;
}
else {
a[i] = x;
b = 0;
}
}
}
void
_PHIPatternWordsBitwiseOr(Int4 *a, Int4 *b, Int4 numWords)
{
Int4 i; /*index over words*/
for (i = 0; i < numWords; i++)
a[i] = (a[i] | b[i]);
}
Int4
_PHIPatternWordsBitwiseAnd(Int4 *result, Int4 *a, Int4 *b, Int4 numWords)
{
Int4 i; /*index over words*/
Int4 returnValue = 0;
for (i = 0; i < numWords; i++) {
if ((result[i] = (a[i] & b[i])))
returnValue = 1;
}
return returnValue;
}
/** Returns the difference between the offset F of a first 1-bit in a word
* sequence and the first offset G < F of a 1-bit in the pattern mask. If
* such G does not exist, it is set to -1.
* @param s Input sequence [in]
* @param mask Array of word masks [in]
* @param numWords Number of words in s. [in]
* @return F - G, see explanation above.
*/
static Int4
s_LenOfL(Int4 *s, Int4 *mask, Int4 numWords)
{
Int4 bitIndex; /*loop index over bits in a word*/
Int4 wordIndex; /*loop index over words*/
Int4 firstOneInMask;
firstOneInMask = -1;
for (wordIndex = 0; wordIndex < numWords; wordIndex++) {
for (bitIndex = 0; bitIndex < PHI_BITS_PACKED_PER_WORD; bitIndex++) {
if ((s[wordIndex] >> bitIndex) % 2 == 1)
return wordIndex*PHI_BITS_PACKED_PER_WORD+bitIndex-firstOneInMask;
if ((mask[wordIndex] >> bitIndex) %2 == 1)
firstOneInMask = wordIndex*PHI_BITS_PACKED_PER_WORD+bitIndex;
}
}
/* This point should never be reached. */
return -1;
}
/** Finds places where pattern matches seq and returns them as
* pairs of positions in consecutive entries of hitArray;
* similar to _PHIBlastFindHitsShort
* @param hitArray Array of hits to return [out]
* @param seq Input sequence [in]
* @param len1 Length of seq [in]
* @param pattern_blk carries all the pattern variables
* @return twice the number of hits.
*/
static Int4
s_FindHitsLong(Int4 *hitArray, const Uint1* seq, Int4 len1,
const SPHIPatternSearchBlk *pattern_blk)
{
Int4 wordIndex; /*index on words in mask*/
Int4 i; /*loop index on seq */
Int4 *prefixMatchedBitPattern; /*see similar variable in
_PHIBlastFindHitsShort*/
Int4 twiceNumHits = 0; /*counter for hitArray*/
Int4 *mask; /*local copy of match_maskL version of pattern
indicates after which positions a match can be declared*/
Int4 *matchResult; /*Array of words to hold the result of the
final test for a match*/
SLongPatternItems* pattern_items = pattern_blk->multi_word_items;
Int4 num_words = pattern_items->numWords;
matchResult = (Int4 *) calloc(num_words, sizeof(Int4));
mask = (Int4 *) calloc(num_words, sizeof(Int4));
prefixMatchedBitPattern = (Int4 *) calloc(num_words, sizeof(Int4));
for (wordIndex = 0; wordIndex < num_words; wordIndex++) {
mask[wordIndex] = pattern_items->match_maskL[wordIndex];
prefixMatchedBitPattern[wordIndex] = 0;
}
/* This is a multiword version of the algorithm in _PHIBlastFindHitsShort */
_PHIPatternWordsLeftShift(mask, 1, num_words);
for (i = 0; i < len1; i++) {
_PHIPatternWordsLeftShift(prefixMatchedBitPattern, 0, num_words);
_PHIPatternWordsBitwiseOr(prefixMatchedBitPattern, mask, num_words);
_PHIPatternWordsBitwiseAnd(prefixMatchedBitPattern, prefixMatchedBitPattern,
pattern_items->bitPatternByLetter[seq[i]],
num_words);
if (_PHIPatternWordsBitwiseAnd(matchResult, prefixMatchedBitPattern,
pattern_items->match_maskL, num_words)) {
hitArray[twiceNumHits++] = i;
hitArray[twiceNumHits++] = i -
s_LenOfL(matchResult, pattern_items->match_maskL, num_words) + 1;
}
}
sfree(prefixMatchedBitPattern);
sfree(matchResult);
sfree(mask);
return twiceNumHits;
}
/** Find matches when pattern is very long,
* @param hitArray Array to pass back pairs of start and end positions for
* hits [out]
* @param seq Input sequence [in]
* @param len Length of seq [in]
* @param is_dna Is sequence DNA or protein? [in]
* @param pattern_blk carries all the pattern variables [in]
* @return Twice the number of hits found.
*/
static Int4
s_FindHitsVeryLong(Int4 *hitArray, const Uint1* seq, Int4 len, Boolean is_dna,
const SPHIPatternSearchBlk *pattern_blk)
{
Int4 twiceNumHits; /*twice the number of matches*/
Int4 twiceHitsOneCall; /*twice the number of hits in one call to
_PHIBlastFindHitsShort */
Int4 wordIndex; /*index over words in pattern*/
Int4 start; /*start position in sequence for calls to _PHIBlastFindHitsShort */
Int4 hitArray1[PHI_MAX_HIT]; /*used to get hits against different words*/
Int4 nextPosInHitArray; /*next available position in hitArray1 */
Int4 hitIndex1, hitIndex2; /*indices over hitArray1*/
SLongPatternItems* multiword_items = pattern_blk->multi_word_items;
SShortPatternItems* word_items = pattern_blk->one_word_items;
SExtraLongPatternItems* extra_items = multiword_items->extra_long_items;
Int4 most_specific_word = extra_items->whichMostSpecific;
word_items->whichPositionPtr = multiword_items->SLL[most_specific_word];
word_items->match_mask = multiword_items->match_maskL[most_specific_word];
if (is_dna) {
word_items->dna_items->DNAwhichPrefixPosPtr =
multiword_items->dna_items->DNAprefixSLL[most_specific_word];
word_items->dna_items->DNAwhichSuffixPosPtr =
multiword_items->dna_items->DNAsuffixSLL[most_specific_word];
}
/*find matches to most specific word of pattern*/
twiceNumHits =
s_FindHitsShortHead(hitArray, seq, 0, len, is_dna, pattern_blk);
if (twiceNumHits < 2)
return 0;
/*extend matches word by word*/
for (wordIndex = most_specific_word+1;
wordIndex < multiword_items->numWords; wordIndex++) {
word_items->whichPositionPtr = multiword_items->SLL[wordIndex];
word_items->match_mask = multiword_items->match_maskL[wordIndex];
if (is_dna) {
word_items->dna_items->DNAwhichPrefixPosPtr =
multiword_items->dna_items->DNAprefixSLL[wordIndex];
word_items->dna_items->DNAwhichSuffixPosPtr =
multiword_items->dna_items->DNAsuffixSLL[wordIndex];
}
nextPosInHitArray = 0;
for (hitIndex2 = 0; hitIndex2 < twiceNumHits; hitIndex2 += 2) {
twiceHitsOneCall =
s_FindHitsShortHead(&hitArray1[nextPosInHitArray], seq,
hitArray[hitIndex2]+1,
MIN(len-hitArray[hitIndex2]-1,
extra_items->spacing[wordIndex-1] +
extra_items->numPlacesInWord[wordIndex]),
is_dna, pattern_blk);
for (hitIndex1 = 0; hitIndex1 < twiceHitsOneCall; hitIndex1+= 2) {
hitArray1[nextPosInHitArray+hitIndex1] =
hitArray[hitIndex2]+hitArray1[nextPosInHitArray+hitIndex1]+1;
hitArray1[nextPosInHitArray+hitIndex1+1] = hitArray[hitIndex2+1];
}
nextPosInHitArray += twiceHitsOneCall;
}
twiceNumHits = nextPosInHitArray;
if (twiceNumHits < 2)
return 0;
/*copy back matches that extend */
for (hitIndex2 = 0; hitIndex2 < nextPosInHitArray; hitIndex2++)
hitArray[hitIndex2] = hitArray1[hitIndex2];
}
/*extend each match back one word at a time*/
for (wordIndex = most_specific_word-1; wordIndex >=0;
wordIndex--) {
word_items->whichPositionPtr = multiword_items->SLL[wordIndex];
word_items->match_mask = multiword_items->match_maskL[wordIndex];
if (is_dna) {
word_items->dna_items->DNAwhichPrefixPosPtr =
multiword_items->dna_items->DNAprefixSLL[wordIndex];
word_items->dna_items->DNAwhichSuffixPosPtr =
multiword_items->dna_items->DNAsuffixSLL[wordIndex];
}
nextPosInHitArray = 0;
for (hitIndex2 = 0; hitIndex2 < twiceNumHits; hitIndex2 += 2) {
start = hitArray[hitIndex2+1] - extra_items->spacing[wordIndex] -
extra_items->numPlacesInWord[wordIndex];
if (start < 0)
start = 0;
twiceHitsOneCall =
s_FindHitsShortHead(&hitArray1[nextPosInHitArray], seq, start,
hitArray[hitIndex2+1]-start, is_dna, pattern_blk);
for (hitIndex1 = 0; hitIndex1 < twiceHitsOneCall; hitIndex1+= 2) {
hitArray1[nextPosInHitArray+hitIndex1] = hitArray[hitIndex2];
hitArray1[nextPosInHitArray+hitIndex1+1] = start +
hitArray1[nextPosInHitArray+hitIndex1+1];
}
nextPosInHitArray += twiceHitsOneCall;
}
twiceNumHits = nextPosInHitArray;
if (twiceNumHits < 2)
return 0;
/*copy back matches that extend*/
for (hitIndex2 = 0; hitIndex2 < nextPosInHitArray; hitIndex2++)
hitArray[hitIndex2] = hitArray1[hitIndex2];
}
return twiceNumHits;
}
Int4 FindPatternHits(Int4 * hitArray, const Uint1* seq, Int4 len,
Boolean is_dna, const SPHIPatternSearchBlk * pattern_blk)
{
if (pattern_blk->flagPatternLength == eOneWord)
return s_FindHitsShortHead(hitArray, seq, 0, len, is_dna, pattern_blk);
if (pattern_blk->flagPatternLength == eMultiWord)
return s_FindHitsLong(hitArray, seq, len, pattern_blk);
return s_FindHitsVeryLong(hitArray, seq, len, is_dna, pattern_blk);
}
SPHIQueryInfo* SPHIQueryInfoNew()
{
SPHIQueryInfo* pattern_info;
const Int4 kMinPhiLookupSize = 8; /* Minimal allocation size for the array
of pattern occurrences. */
if ((pattern_info =
(SPHIQueryInfo*) calloc(1, sizeof(SPHIQueryInfo))) == NULL)
return NULL;
pattern_info->allocated_size = kMinPhiLookupSize;
if ((pattern_info->occurrences =
(SPHIPatternInfo*) calloc(kMinPhiLookupSize, sizeof(SPHIPatternInfo)))
== NULL)
return NULL;
return pattern_info;
}
SPHIQueryInfo*
SPHIQueryInfoFree(SPHIQueryInfo* pat_info)
{
if (pat_info) {
sfree(pat_info->occurrences);
sfree(pat_info->pattern);
sfree(pat_info);
}
return NULL;
}
SPHIQueryInfo*
SPHIQueryInfoCopy(const SPHIQueryInfo* pat_info)
{
SPHIQueryInfo* retval = NULL;
if (!pat_info)
return retval;
retval =
(SPHIQueryInfo*) BlastMemDup(pat_info, sizeof(SPHIQueryInfo));
retval->pattern =
(char *) BlastMemDup(pat_info->pattern, 1+strlen(pat_info->pattern));
retval->occurrences = (SPHIPatternInfo*)
BlastMemDup(pat_info->occurrences,
pat_info->num_patterns*sizeof(SPHIPatternInfo));
return retval;
}
/** Adds a new pattern hit to the PHI BLAST pseudo lookup table.
* @param pattern_info The query pattern information structure. [in] [out]
* @param offset Offset in query at which pattern was found. [in]
* @param length Length of the pattern at this offset. [in]
*/
static Int2
s_PHIBlastAddPatternHit(SPHIQueryInfo* pattern_info,
Int4 offset, Int4 length)
{
SPHIPatternInfo* occurrence_array;
Int4 pat_index = pattern_info->num_patterns;
if (pat_index >= pattern_info->allocated_size) {
if ((occurrence_array = (SPHIPatternInfo*)
realloc(pattern_info->occurrences,
2*pattern_info->allocated_size*sizeof(SPHIPatternInfo)))
== NULL)
return -1;
pattern_info->occurrences = occurrence_array;
pattern_info->allocated_size *= 2;
}
pattern_info->occurrences[pat_index].offset = offset;
pattern_info->occurrences[pat_index].length = length;
++pattern_info->num_patterns;
return 0;
}
Int4 PHIGetPatternOccurrences(const SPHIPatternSearchBlk * pattern_blk,
const BLAST_SequenceBlk * query,
const BlastSeqLoc * location,
Boolean is_dna,
BlastQueryInfo * query_info)
{
const BlastSeqLoc* loc;
Int4* hitArray;
EBlastProgramType program = (is_dna ? eBlastTypePhiBlastn : eBlastTypePhiBlastp);
SPHIQueryInfo* pattern_info = query_info->pattern_info;
ASSERT(pattern_info);
hitArray = (Int4 *) calloc(2*query->length, sizeof(Int4));
for(loc=location; loc; loc=loc->next) {
Int4 i, twiceNumHits;
Int4 from, to;
Int4 loc_length;
Uint1* sequence;
from = loc->ssr->left;
to = loc->ssr->right;
loc_length = to - from + 1;
sequence = query->sequence + from;
twiceNumHits = FindPatternHits(hitArray, sequence, loc_length, is_dna,
pattern_blk);
for (i = 0; i < twiceNumHits; i += 2) {
if (hitArray[i+1]+from == 0 &&
hitArray[i]-hitArray[i+1]+1 == BlastQueryInfoGetQueryLength(query_info, program, 0))
{
return INT4_MAX;
}
s_PHIBlastAddPatternHit(pattern_info, hitArray[i+1]+from,
hitArray[i]-hitArray[i+1]+1);
}
}
sfree(hitArray);
return pattern_info->num_patterns;
}