Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Hyperseti 1.2.0 #115

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ config = {
'sk_flag': True, # Apply spectral kurtosis flagging
'normalize': True, # Normalize data
'blank_edges': {'n_chan': 32}, # Blank edges channels
'blank_extrema': {'threshold': 10000} # Blank ridiculously bright signals before search
'blank_extrema': {'threshold': 10000}, # Blank ridiculously bright signals before search
'poly_fit': 3 # Subtract a 3-order polynomial bandpass
},
'dedoppler': {
'kernel': 'ddsk', # Doppler + kurtosis doppler (ddsk)
Expand All @@ -47,7 +48,13 @@ config = {
'threshold': 20, # SNR threshold above which to consider a hit
},
'pipeline': {
'merge_boxcar_trials': True # Merge hits at same frequency that are found in multiple boxcars
'merge_boxcar_trials': True, # Merge hits at same frequency that are found in multiple boxcars
'blank_hits':
{
'n_blank': 4, # Do 4 rounds of iterative blanking
'padding': 16 # Blank signal + 16 neighboring bins
}

}
}

Expand Down
21 changes: 6 additions & 15 deletions docs/pipeline/hitsearch.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,22 +56,13 @@ An example of iterative blanking is show below; in the example, the highest S/N

![Iterative-blanking](https://user-images.githubusercontent.com/713251/227689177-42e81c48-53cc-4eb9-a8f9-4cea8ce37f2e.png)

Iterative blanking can be enabled by setting `config['pipeline']['n_blank']` to 1 or greater, where `n_blank` is the
number of iterations to apply. Note that if no new hits are found after blanking, the process will stop.
Iterative blanking can be enabled by setting `config['pipeline']['n_blank']` to 1 or greater, where `n_blank` is the number of iterations to apply. Note that if no new hits are found after blanking, the process will stop.

### Browsing hits

Results from `find_et` are returned as a `HitBrowser`, which has a `view_hits()` method for viewing this, and an `extract_hits()` method for extracting hits.

A Pandas DataFrame of all hits found is attached as `hit_brower.hit_table`, and the data are accessible via `hit_browser.data_array`.

```python
hit_browser = find_et(voyager_h5, config, gulp_size=2**20)
display(hit_browser.hit_table)

hit_browser.view_hit(0, padding=128, plot='dual')
``` python
config['pipeline']['blank_hits'] = {
'n_blank': 4, # Do 4 rounds of iterative blanking
'padding': 16 # Blank signal + 16 neighboring bins
}
```

![image](https://user-images.githubusercontent.com/713251/227728999-1bec6e2f-bfca-4ab7-ae59-d08010ad8a8d.png)


14 changes: 8 additions & 6 deletions docs/usage/quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,23 +40,25 @@ config = {
'normalize': True, # Normalize data
'blank_edges': {'n_chan': 32}, # Blank edges channels
'blank_extrema': {'threshold': 10000} # Blank ridiculously bright signals before search
'poly_fit': 5
'poly_fit': 5 # Subtract a 5-order polynomial bandpass
},
'dedoppler': {
'kernel': 'ddsk', # Doppler + kurtosis doppler (ddsk)
'max_dd': 10.0, # Maximum dedoppler delay, 5 Hz/s
'min_dd': None, #
'max_dd': 10.0, # Maximum dedoppler delay, 10 Hz/s
'min_dd': None, # If None, uses max_dd * -1
'apply_smearing_corr': True , # Correct for smearing within dedoppler kernel
# Note: set to False if using multiple boxcars
'plan': 'stepped' # Dedoppler trial spacing plan (stepped = less memory)
},
'hitsearch': {
'threshold': 20, # SNR threshold above which to consider a hit
'min_fdistance': None # Automatic calculation of min. channel spacing between hits
},
'pipeline': {
'n_boxcar': 10, # Number of boxcar trials to apply (10 stages, 2^10 channels)
# Boxcar is a moving average to compensate for smearing / broadband
'blank_hits':
{
'n_blank': 4, # Do 4 rounds of iterative blanking
'padding': 16 # Blank signal + 16 neighboring bins
}
'merge_boxcar_trials': True # Merge hits at same frequency that are found in multiple boxcars
}
}
Expand Down
7 changes: 5 additions & 2 deletions hyperseti/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ def __init__(self, data_array: DataArray, config: dict, gpu_id: int=None, kernel
self.data_array = data_array
self.config = deepcopy(config)
self._called_count = 0
self._blank_count = 0

if not isinstance(self.data_array.data, cp.ndarray):
logger.warning(f"GulpPipeline init: Data not in cupy.ndarray, attempting to copy data to GPU")
Expand Down Expand Up @@ -184,7 +185,7 @@ def hitsearch(self):
# sqrt(N_timesteps) is taken into account within dedoppler kernel
conf = deepcopy(self.config) # This deepcopy avoids overwriting original threshold value
boxcar_size = conf['dedoppler'].get('boxcar_size', 1)
blank_count = conf['pipeline'].get('blank_count', 1)
blank_count = self._blank_count

_threshold0 = conf['hitsearch']['threshold']
#conf['hitsearch']['threshold'] = _threshold0 * np.sqrt(boxcar_size)
Expand Down Expand Up @@ -239,6 +240,7 @@ def run(self) -> pd.DataFrame:
n_hits_last_iter = 0

for blank_count in range(n_blank):
self._blank_count = blank_count
new_peaks_this_blanking_iter = []
n_hits_blanking_iter = 0
for boxcar_idx, boxcar_size in enumerate(boxcar_trials):
Expand Down Expand Up @@ -274,7 +276,8 @@ def run(self) -> pd.DataFrame:
else:
logger.info(f"GulpPipeline.run #{self._called_count}: Elapsed time: {(t1-t0):2.2f}s; {len(self.peaks)} hits found")

self.peaks['n_blank'] = n_blank
self.peaks['n_blank'] = n_blank


return self.peaks

Expand Down