diff --git a/README.md b/README.md index d6a45f9..ed844de 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,8 @@ config = { 'sk_flag': True, # Apply spectral kurtosis flagging 'normalize': True, # Normalize data 'blank_edges': {'n_chan': 32}, # Blank edges channels - 'blank_extrema': {'threshold': 10000} # Blank ridiculously bright signals before search + 'blank_extrema': {'threshold': 10000}, # Blank ridiculously bright signals before search + 'poly_fit': 3 # Subtract a 3-order polynomial bandpass }, 'dedoppler': { 'kernel': 'ddsk', # Doppler + kurtosis doppler (ddsk) @@ -47,7 +48,13 @@ config = { 'threshold': 20, # SNR threshold above which to consider a hit }, 'pipeline': { - 'merge_boxcar_trials': True # Merge hits at same frequency that are found in multiple boxcars + 'merge_boxcar_trials': True, # Merge hits at same frequency that are found in multiple boxcars + 'blank_hits': + { + 'n_blank': 4, # Do 4 rounds of iterative blanking + 'padding': 16 # Blank signal + 16 neighboring bins + } + } } diff --git a/docs/pipeline/hitsearch.md b/docs/pipeline/hitsearch.md index 23e88ab..4c9fdb9 100644 --- a/docs/pipeline/hitsearch.md +++ b/docs/pipeline/hitsearch.md @@ -56,22 +56,13 @@ An example of iterative blanking is show below; in the example, the highest S/N ![Iterative-blanking](https://user-images.githubusercontent.com/713251/227689177-42e81c48-53cc-4eb9-a8f9-4cea8ce37f2e.png) -Iterative blanking can be enabled by setting `config['pipeline']['n_blank']` to 1 or greater, where `n_blank` is the -number of iterations to apply. Note that if no new hits are found after blanking, the process will stop. +Iterative blanking can be enabled by setting `config['pipeline']['n_blank']` to 1 or greater, where `n_blank` is the number of iterations to apply. Note that if no new hits are found after blanking, the process will stop. -### Browsing hits - -Results from `find_et` are returned as a `HitBrowser`, which has a `view_hits()` method for viewing this, and an `extract_hits()` method for extracting hits. - -A Pandas DataFrame of all hits found is attached as `hit_brower.hit_table`, and the data are accessible via `hit_browser.data_array`. - -```python -hit_browser = find_et(voyager_h5, config, gulp_size=2**20) -display(hit_browser.hit_table) - -hit_browser.view_hit(0, padding=128, plot='dual') +``` python +config['pipeline']['blank_hits'] = { + 'n_blank': 4, # Do 4 rounds of iterative blanking + 'padding': 16 # Blank signal + 16 neighboring bins + } ``` -![image](https://user-images.githubusercontent.com/713251/227728999-1bec6e2f-bfca-4ab7-ae59-d08010ad8a8d.png) - diff --git a/docs/usage/quickstart.md b/docs/usage/quickstart.md index 66cb259..fde468c 100644 --- a/docs/usage/quickstart.md +++ b/docs/usage/quickstart.md @@ -40,14 +40,13 @@ config = { 'normalize': True, # Normalize data 'blank_edges': {'n_chan': 32}, # Blank edges channels 'blank_extrema': {'threshold': 10000} # Blank ridiculously bright signals before search - 'poly_fit': 5 + 'poly_fit': 5 # Subtract a 5-order polynomial bandpass }, 'dedoppler': { 'kernel': 'ddsk', # Doppler + kurtosis doppler (ddsk) - 'max_dd': 10.0, # Maximum dedoppler delay, 5 Hz/s - 'min_dd': None, # + 'max_dd': 10.0, # Maximum dedoppler delay, 10 Hz/s + 'min_dd': None, # If None, uses max_dd * -1 'apply_smearing_corr': True , # Correct for smearing within dedoppler kernel - # Note: set to False if using multiple boxcars 'plan': 'stepped' # Dedoppler trial spacing plan (stepped = less memory) }, 'hitsearch': { @@ -55,8 +54,11 @@ config = { 'min_fdistance': None # Automatic calculation of min. channel spacing between hits }, 'pipeline': { - 'n_boxcar': 10, # Number of boxcar trials to apply (10 stages, 2^10 channels) - # Boxcar is a moving average to compensate for smearing / broadband + 'blank_hits': + { + 'n_blank': 4, # Do 4 rounds of iterative blanking + 'padding': 16 # Blank signal + 16 neighboring bins + } 'merge_boxcar_trials': True # Merge hits at same frequency that are found in multiple boxcars } } diff --git a/hyperseti/pipeline.py b/hyperseti/pipeline.py index 67a4d97..199ecef 100644 --- a/hyperseti/pipeline.py +++ b/hyperseti/pipeline.py @@ -73,6 +73,7 @@ def __init__(self, data_array: DataArray, config: dict, gpu_id: int=None, kernel self.data_array = data_array self.config = deepcopy(config) self._called_count = 0 + self._blank_count = 0 if not isinstance(self.data_array.data, cp.ndarray): logger.warning(f"GulpPipeline init: Data not in cupy.ndarray, attempting to copy data to GPU") @@ -184,7 +185,7 @@ def hitsearch(self): # sqrt(N_timesteps) is taken into account within dedoppler kernel conf = deepcopy(self.config) # This deepcopy avoids overwriting original threshold value boxcar_size = conf['dedoppler'].get('boxcar_size', 1) - blank_count = conf['pipeline'].get('blank_count', 1) + blank_count = self._blank_count _threshold0 = conf['hitsearch']['threshold'] #conf['hitsearch']['threshold'] = _threshold0 * np.sqrt(boxcar_size) @@ -239,6 +240,7 @@ def run(self) -> pd.DataFrame: n_hits_last_iter = 0 for blank_count in range(n_blank): + self._blank_count = blank_count new_peaks_this_blanking_iter = [] n_hits_blanking_iter = 0 for boxcar_idx, boxcar_size in enumerate(boxcar_trials): @@ -274,7 +276,8 @@ def run(self) -> pd.DataFrame: else: logger.info(f"GulpPipeline.run #{self._called_count}: Elapsed time: {(t1-t0):2.2f}s; {len(self.peaks)} hits found") - self.peaks['n_blank'] = n_blank + self.peaks['n_blank'] = n_blank + return self.peaks