
    VKjv!              	      8   d Z ddlmZ ddlZddlZddlmZ ddlmZ ddl	m
Z
mZ ddlmZmZ d	Zd
Zd\  ZZZdZdZ ej,                  d      Z ej,                  de de dej0                        Z ej,                  de dej0                        Z ej,                  de dej0                        Z ej,                  dej0                        Z ej,                  d      Zd dZd!dZd dZ d"dZ!d#dZ"d$dZ#d%dZ$ddddddddd	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d&dZ%y)'aQ  Phrase suggestions for bottom lines and problem discussions.

`rebuild` splits every stored bottom line and problem discussion for a center
into sentences and writes them to `phrases` with the forecast's context
(problem types, likelihood, size, locations, date). `suggest` ranks sentences
for a draft by how closely their forecast matches it and by how often the
wording was reused across forecasts.

Danger-level words are masked to "[rating]" so a suggestion never sets the
rating for the forecaster. Imported archives are legacy-era text: descriptive
wording to borrow from, not guidance.
    )annotationsN)date)
Connection   )fetchallfetchone)_jaccard_season_closeness)bottom_lineproblemz[rating])   i     i N  z*(?:low|moderate|considerable|high|extreme)z.\b(?:LOW|MODERATE|CONSIDERABLE|HIGH|EXTREME)\bz\bz#(?=(?:[\s,]+(?:to|and|or|avalanche|z|\(\d\)))*[\s,]+danger\b)a$  (\bdanger\s+(?:(?:is|are|be|remains?|will|should|could|may|stays?|also|again|back|still|rated|at|near|mostly|generally|becomes?|increases?|rises?|climbs?|reach(?:es)?|decreases?|drops?|returns?|ranges?|from|to|up|down|for|today|tonight|tomorrow|quickly|slowly|gradually|likely|expected)\s+)+)z&(\[rating\],?\s+(?:to|and|or|then)\s+)z[\b(?:mt|mtn|st|hwy|approx|e\.g|i\.e|ft|vs|jan|feb|mar|apr|jun|jul|aug|sept?|oct|nov|dec)\.$u"   (?<=[.!?])\s+(?=[\"“(]?[A-Z0-9])c                    t         j                  t        |       } t        j                  t        |       } t        j                  d |       } t
        j                  d |       S )Nc                2    | j                  d      t        z   S Nr   groupRATINGms    (/var/www/avy-guidance/api/app/phrases.py<lambda>zmask_danger.<locals>.<lambda>5   s    AGGAJ,?     c                2    | j                  d      t        z   S r   r   r   s    r   r   zmask_danger.<locals>.<lambda>6   s    aggaj6&9 r   )_CAPS_LEVELsubr   _LEVEL_BEFORE_DANGER_DANGER_VERB_LEVEL_RATING_PAIR)texts    r   mask_dangerr!   2   sK    ??64(D##FD1D!!"?FD94@@r   c                n   | sg S g }t        j                  d| j                  dd            D ]  }dj                  |j                               }d}t        j                  |      D ]6  }|r| d| n|}t
        j                  |      r$|j                  |       d}8 |su|j                  |        |S )Nz\n\s*\n  )resplitreplacejoin_SENTENCE_END_ABBREVsearchappend)r    outparabufparts        r   split_sentencesr2   9   s    	
CT\\$%;< 	xx

%!''- 	D%(SE4&/dC>>#&

3		
 JJsO	 Jr   c                    t        j                  dd| j                               }dj                  t        j                  dd|      j	                               S )z3Grouping key: case, digits and punctuation ignored.z\d+(?:\.\d+)?#r%   z[^a-z#\[\]]+)r&   r   lowerr)   r'   )r    ts     r   	normalizer7   J   s@    
djjl3A88BFF?C399;<<r   c                B   t               g }}t        |       D ]  }t        t        |      cxk  r	t        k  r#n n t        |j                               t        k  rCt        |      }t        |      }|s\||vsa|j                  |       |j                  ||f        |S )z;Usable (text, norm) sentences from one field, deduplicated.)setr2   MIN_LENlenMAX_LENr'   	MIN_WORDSr!   r7   addr-   )r    seenr.   sns        r   phrases_fromrB   P   s    r#DT" #a&+G+s1779~	/INaL$HHQKJJ1v Jr   c                f   | j                  d|f       g }t        | d|f      D ]D  }t        |d         D ]1  \  }}|j                  ||d   d|d   ddg |||d   |d   |d	   f       3 F t        | d
|f      D ]N  }t        |d         D ];  \  }}|j                  ||d   d|d   g|d   |d   |d   |||d   |d   |d	   f       = P | j	                         5 }|j                  d|       ddd       t        |      t        |D ch c]  }|d   |d   f c}      dS # 1 sw Y   6xY wc c}w )z>Replace the center's phrase library from its stored forecasts.z(DELETE FROM phrases WHERE center_id = %sa3  SELECT f.id, f.valid_date, f.source, f.guidance_era, f.bottom_line,
                  coalesce(array_agg(DISTINCT p.problem_type) FILTER (WHERE p.id IS NOT NULL), '{}') AS types
           FROM forecasts f LEFT JOIN forecast_problems p ON p.forecast_id = f.id
           WHERE f.center_id = %s GROUP BY f.idr   idtypesN
valid_datesourceguidance_erazSELECT p.forecast_id, p.problem_type, p.likelihood, p.size_max, p.locations, p.discussion,
                  f.valid_date, f.source, f.guidance_era
           FROM forecast_problems p JOIN forecasts f ON f.id = p.forecast_id WHERE f.center_id = %s
discussionforecast_idr   problem_type
likelihoodsize_max	locationszINSERT INTO phrases (center_id, forecast_id, field, problem_types, likelihood, size_max, locations,
                   text, norm, valid_date, source, guidance_era)
               VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)      )phrasesdistinct)executer   rB   r-   cursorexecutemanyr;   )	conn	center_idrowsfr    normpcurrs	            r   rebuildr^   ^   s   LL;i\JD	3 
 
K 'q'78 	KJD$KKAdG]AgJdTVX\^b<!H+q7HJ K	K
K 	g 
 	g 'q7 	gJD$KKAm$4i!NBSATVWXdVeghisgt;tQ|_akSTUcSdf g	g	g 
 
#J 		

 4yc2NAAaD!A$<2N.OPP
 
 3Os   D"D.
"D+c                    t        j                  d| xs dj                               d d }dj                  d |D              xs d S )Nz	[a-z0-9]+r$   rP   z & c              3  &   K   | ]	  }| d   yw)z:*N ).0ws     r   	<genexpr>z"_prefix_tsquery.<locals>.<genexpr>   s     .12h.s   )r&   findallr5   r)   )qwordss     r   _prefix_tsqueryrh      s>    JJ|ag2__%67;E::...6$6r   c           	        |rdt        || d         z  nd}|dk(  r$||rdt        |t        | d               z  z   S dz   S |r|t        |t        | d   xs g             z  }|r.| d   r)|d	t        t	        |      t	        | d         z
        z  z  }|.| d
   )|d	t        t        |      t        | d
         z
        z  z  }|S )Ng      ?rF   g        r   g       @problem_typesrN   rL   g?rM   )r
   r	   r9   absintfloat)rowfieldrE   rL   rM   rN   rF   r@   s           r   _contextrp      s    BL
C,=>>RUAC(5#c/.B*CDDWWSVWW	XiS%5%;!<==c,'	S3s:S->)??@@@J ;	S3uXs:)??@@@Hr      )rK   rj   rL   rM   rN   rF   rf   limitc               p   ddg||g}}t        |xs g       }|dk(  r&|h}|j                  d       |j                  |       n-|r+|j                  d       |j                  t        |             t        |	      }|r"|j                  d       |j                  |       t	        | ddj                  |       d	t         |      }t        |xs g       }i }|D ]  }t        |||||||      }|j                  |d
         }|*|d   t               |d   |d   t               |dx||d
   <   }|d   j                  |d          |d   j                  |d          t        |d   |      |d<    g }|j                         D ]h  }t        |d         }|j                  |d   ||d   |d   |d   dhk(  t        |d   v t        |d   dt        j                   |      z  z   d      d       j |j#                  d        | xr t%        | d|f       }|d t        dt'        |
d             t        |      |dS )Nzcenter_id = %sz
field = %sr   z&problem_types @> ARRAY[%s]::smallint[]zproblem_types && %s::smallint[]z tsv @@ to_tsquery('english', %s)zSELECT forecast_id, problem_types, likelihood, size_max, locations, text, norm, valid_date, guidance_era
            FROM phrases WHERE z AND z  ORDER BY valid_date DESC LIMIT rZ   r    rF   rJ   )r    forecast_ids	last_usedexample_forecast_iderascontextrt   rw   rH   rx   ru   rv   legacyg?   )r    usesru   rv   ry   
has_ratingscorec                    | d    | d    fS )Nr}   r{   ra   )is    r   r   zsuggest.<locals>.<lambda>   s    qzkAfI:6 r   )keyz2SELECT 1 FROM phrases WHERE center_id = %s LIMIT 1r   2   )itemsmatchedlibrary_empty)r9   r-   sortedrh   r   r)   MAX_CANDIDATESrp   getr>   maxvaluesr;   r   roundmathlog1psortr   min)rV   rW   ro   rK   rj   rL   rM   rN   rF   rf   rr   whereparamsrE   tsqrX   locsgroupsr]   ctxgr   rA   emptys                           r   suggestr      sy    &|4y%6H6E#$E	=>l#	67fUm$
!
C
78c  'U 344TUcTd	h	D yBD F .q%
HdJOJJqy!9-.vY\]^j\k<=m<LVYV[hk%m mF1V9	.a./	&	a'(1Y<-). E]]_ .!"fIqq~^_`u^viH:-Vqy=P1Y<#

1*==qA
 	 
JJ6J7HoXd,`clbnooE3SCrN34V_deer   )r    strreturnr   )r    
str | Noner   z	list[str])r    r   r   zlist[tuple[str, str]])rV   r   rW   rl   r   dict)rf   r   r   r   )rn   r   ro   r   rE   zset[int]rN   zset[str]rF   date | Noner   rm   )rV   r   rW   rl   ro   r   rK   
int | Nonerj   zlist[int] | NonerL   r   rM   zfloat | NonerN   zlist[str] | NonerF   r   rf   r   rr   rl   r   r   )&__doc__
__future__r   r   r&   datetimer   psycopgr   dbr   r   	retrievalr	   r
   FIELDSr   r:   r<   r=   r   _LEVELcompiler   Ir   r   r   r+   r*   r!   r2   r7   rB   r^   rh   rp   r   ra   r   r   <module>r      s   #  	   " 2	#	( )	6bjjJK!rzz	&4VH<UVXZX\X\   RZZj 	xr DD  rzzCF82NPRPTPTU
"**suwuyuy
z

@AA"= QF7

 Y].2Tlp*.$`d,f+,f@J,f]i,f',f<G,fS],f ,f !%,fr   