o [>e'@s&ddlZejdejdddlmZmZddlmZddlZ ddl m Z ddl m Z ddlmZmZddlZdd lmZGd d d Zed krdd lmZmZmZeeeeZedZeeejZdZdZ dZ!e"eee e!\Z#Z$ee$e#Z%eeej&Ze'de%e'de$dSdS)Nz..z./sam)sam_model_registrySamAutomaticMaskGenerator)get_aot) Segmentor)Detector) draw_outline draw_points) draw_maskc@seZdZd(ddZddZddZd d Zd)d d Zd*ddZddZ ddZ ddZ ddZ de jde jfddZd+de jde jde jfd d!Zd"e jfd#d$Zd,de jfd&d'ZdS)- SegTrackerreturnNcCst||_t||_t|jj|_|d|_|d|_|d|_ |d|_ g|_ d|_ d|_ d|_d|_g|_g|_tddS)z* Initialize SAM and AOT. sam_gapmin_area max_obj_nummin_new_obj_iouNzSegTracker has been initialized)rsamrtrackerrdevicedetectorr r rrreference_objs_list object_idxcurr_idxorigin_merged_maskfirst_frame_maskeverything_pointseverything_labelsprint)selfsegtracker_argssam_argsaot_argsr!>D:\05 Dr\Segmentation\Segment-and-Track-Anything\SegTracker.py__init__s       zSegTracker.__init__cCs:|dddddddf}|jj|}t|dkrdStj|ddjtjd|_d}|D](}|d|j krW|d}||j|dk<|d7}|j |dd|j dq/t |j}||dk}d|_|D],}t|j|k|j ks||j|jkrd|j|j|k<qi|j|j|j|k<|jd7_qi|j|_|jS) z Arguments: frame: numpy array (h,w,3) Return: origin_merged_mask: numpy array (h,w) NrZ segmentationdtyperareaZ point_coords)rZeverything_generatorgeneratelennpzerosshapeuint8rr rappendruniquersumrr)rframeannsidxannmZobj_idsidr!r!r"seg%s0    "zSegTracker.segcCs ||_dSN)r)rZupdated_merged_maskr!r!r"update_origin_merged_maskLs z$SegTracker.update_origin_merged_maskcCs||_||_dSr8)rr)rmaskr6r!r!r"reset_origin_merged_maskRs z#SegTracker.reset_origin_merged_maskrcCs<|jt||d|_|j|||jd|dS)z Add objects in a mask for tracking. Arguments: frame: numpy array (h,w,3) mask: numpy array (h,w) rN)rr.r*r/ get_obj_numrrZadd_reference_frame)rr1r:Z frame_stepr!r!r" add_referenceVszSegTracker.add_referenceFcCs@|j|}|r|j||ddtj S)z Track all known objects. Arguments: frame: numpy array (h,w,3) Return: origin_merged_mask: numpy array (h,w) r) rtrack update_memorysqueezedetachZcpunumpyastyper*r-)rr1r? pred_maskr!r!r"r>as  $zSegTracker.trackcCsBt}|jD] }|t|qttt|}dd|D}|S)NcSsg|]}|dkr|qSrr!).0ir!r!r" ssz0SegTracker.get_tracking_objs..)setrupdatelistsorted)robjsrefr!r!r"get_tracking_objsns  zSegTracker.get_tracking_objscCs$|}t|dkr dStt|S)Nr)rOr)intmax)rrMr!r!r"r<vs zSegTracker.get_obj_numc Cs|dk|}t|}||dk}|j}|D]2}t||k}t||k}|||jks7||jks7||jkr>d|||k<q||||k<|d7}q|S)a, Compare tracked results from AOT with segmented results from SAM. Select objects from background if they are not tracked. Arguments: track_mask: numpy array (h,w) seg_mask: numpy array (h,w) Return: new_obj_mask: numpy array (h,w) rr)r*r/rr0rr r) rZ track_maskZseg_maskZ new_obj_maskZ new_obj_idsZobj_numr3Z new_obj_areaZobj_arear!r!r" find_new_objs{s     zSegTracker.find_new_objscCs|jdSr8)rrestart)rr!r!r"restart_trackerszSegTracker.restart_tracker origin_framebboxcCsJ|j||d}||}t||}t||d|dd}||fS)z' Use bbox-prompt to get mask Parameters: origin_frame: H, W, C bbox: [[x0, y0], [x1, y1]] Return: refined_merged_mask: numpy array (h, w) masked_frame: numpy array (h, w, c) rr)rr)rsegment_with_boxadd_maskr copycv2 rectangle)rrUrVinteractive_maskrefined_merged_mask masked_framer!r!r" seg_acc_bboxs  zSegTracker.seg_acc_bboxTcoordsmodescCsH|j||||}||}t||}t|||}t||}||fS)a  Use point-prompt to get mask Parameters: origin_frame: H, W, C coords: nd.array [[x, y]] modes: nd.array [[1]] Return: refined_merged_mask: numpy array (h, w) masked_frame: numpy array (h, w, c) )rZsegment_with_clickrYr rZrr)rrUrarb multimaskr]r^r_r!r!r" seg_acc_clicks    zSegTracker.seg_acc_clickr]cCs:|jdurtj|jtjd|_|j}|j||dk<|S)z Merge interactive mask with self.origin_merged_mask Parameters: interactive_mask: numpy array (h, w) Return: refined_merged_mask: numpy array (h, w) Nr%r)rr*r+r,r-rZr)rr]r^r!r!r"rYs  zSegTracker.add_maskrcCs|j}|j}|j||||\} } tt| D]F} | | } | dd| dd| dd| dd| jd| jd|krBq|j|| |d} | | }| ||jd7_q| |||| fS)z Using Grounding-DINO to detect object acc Text-prompts Retrun: refined_merged_mask: numpy array (h, w) annotated_frame: numpy array (h, w, 3) rr) rrrZ run_groundingranger)r,rrXrYr9r;)rrUgrounding_caption box_thresholdtext_thresholdZbox_size_thresholdZ reset_imageZbc_idZbc_maskannotated_frameboxesrGrVr]r^r!r!r"detect_and_segsH   zSegTracker.detect_and_seg)r NrE)F)T)rF)__name__ __module__ __qualname__r#r7r9r;r=r>rOr<rRrTr*ndarrayr`rdrYrkr!r!r!r"r s '   r __main__)rrr zA/data2/cym/Seg_Tra_any/Segment-and-Track-Anything/debug/point.pngz swan.waterg?z./debug/masked_frame.pngz ./debug/x.png)(syspathr.Zsam.segment_anythingrrZ aot_trackerrrBr*Ztool.segmentorrZ tool.detectorrtool.transfer_toolsrrr[seg_track_anythingr r rl model_argsrrr Seg_TrackerimreadrUcvtColor COLOR_BGR2RGBrfrgrhrkpredicted_maskrir_Z COLOR_RGB2BGRZimwriter!r!r!r"s6      i