Skip to content

Commit

Permalink
Merge pull request #682 from aodn/hotfix_2.6.9
Browse files Browse the repository at this point in the history 
Hotfix 2.6.9
  • Loading branch information
@lbesnard
lbesnard authored Aug 25, 2020
2 parents 7b274d5 + 86c337c commit 64d4289
Show file tree
Hide file tree
Showing 7 changed files with 213 additions and 56 deletions.
110 changes: 73 additions & 37 deletions AutomaticQC/imosTimeSeriesSpikeQC.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,43 +48,29 @@
if (is_profile || time_missing), return, end


if isfield(sample_data.meta,'instrument_burst_duration')
burst_duration = sample_data.meta.instrument_burst_duration;
valid_burst_duration = ~isempty(burst_duration) && ~isnan(burst_duration) && burst_duration>0;
elseif isfield(sample_data,'instrument_burst_duration')
burst_duration = sample_data.meta.instrument_burst_duration;
valid_burst_duration = ~isempty(burst_duration) && ~isnan(burst_duration) && burst_duration>0;
end
if isfield(sample_data.meta,'instrument_burst_interval')
burst_interval = sample_data.meta.instrument_burst_interval;
valid_burst_interval = ~isempty(burst_interval) && ~isnan(burst_interval) && burst_interval>0;
elseif isfield(sample_data,'instrument_burst_interval')
burst_interval = sample_data.meta.instrument_burst_interval;
valid_burst_interval = ~isempty(burst_interval) && ~isnan(burst_interval) && burst_interval>0;
end

has_burst = (valid_burst_duration) || (valid_burst_interval);

if has_burst
[is_burst_sampling,is_burst_metadata_valid,burst_interval] = get_burst_metadata(sample_data);
if is_burst_metadata_valid
opts_file = ['AutomaticQC' filesep 'imosTimeSeriesSpikeQCBurst.txt'];
else
elseif ~is_burst_sampling
opts_file = ['AutomaticQC' filesep 'imosTimeSeriesSpikeQC.txt'];
else
warning('Invalid burst metadata...skipping');
return
end

ts_variables = load_timeseries_variables(sample_data);

if isempty(ts_variables), return, end
postqc_data = gen_postqc(sample_data, ts_variables, burst_interval);

postqc_data = gen_postqc(sample_data, ts_variables, has_burst);

if has_burst && numel(postqc_data.burst_index_range) == 1
if is_burst_sampling && numel(postqc_data.burst_index_range) == 1
warning('Burst metadata not consistent with TIME variable...skipping')
return
end

if auto
user_interaction = struct();
spike_methods = loadSpikeClassifiers(opts_file, has_burst);
spike_methods = loadSpikeClassifiers(opts_file, is_burst_sampling);
method = readProperty('auto_function',opts_file);

for k = 1:length(ts_variables)
Expand All @@ -93,7 +79,7 @@
end

else
wobj = spikeWindow(postqc_data, ts_variables, opts_file, has_burst);
wobj = spikeWindow(postqc_data, ts_variables, opts_file, is_burst_sampling);
uiwait(wobj.ui_window);
user_interaction = wobj.UserData;
end
Expand All @@ -112,7 +98,7 @@

if isempty(user_input.spikes)%run now if preview was not triggered

if has_burst
if is_burst_sampling
user_input.spikes = user_input.fun(postqc_var.valid_burst_range, postqc_var.data, user_input.args{:});
else
user_input.spikes = user_input.fun(postqc_var.data, user_input.args{:});
Expand Down Expand Up @@ -185,7 +171,7 @@

end

function postqc = gen_postqc(sample_data, ts_variables, has_burst)
function postqc = gen_postqc(sample_data, ts_variables, burst_interval)
%function postqc = gen_postqc(sample_data, ts_variables)
%
% Construct a procqc data structure, the input for the SpikeQC window class.
Expand All @@ -195,7 +181,7 @@
%
% sample_data - the toolbox data struct.
% ts_variables - the time-series variables to process.
% has_burst - if the data is in burst-mode.
% burst_interval - the burst interval in seconds. Empty for non-burst data
%
% Outputs:
%
Expand All @@ -206,7 +192,7 @@
% postqc.variables{k}.time = The time values.
% postqc.variables{k}.l - the first index of valid data (from continous invalid ones)
% postqc.variables{k}.r - the last index of valid data (ditto)
% postqc.variables{k}.valid_burst_range = the burst index ranges [only if has_burst is true]
% postqc.variables{k}.valid_burst_range = the burst index ranges [only if burst_interval is not-empty]
%
%
% author: hugo.oliveira@utas.edu.au
Expand All @@ -221,13 +207,7 @@
postqc.variables = cell(1, n_ts_vars);
postqc.dimensions = sample_data.dimensions;

if has_burst
try
burst_interval = sample_data.meta.instrument_burst_interval;
catch
burst_interval = sample_data.instrument_burst_interval;
end

if ~isempty(burst_interval)
postqc.burst_precision = precisionBounds(burst_interval / 86400);
postqc.burst_index_range = findBursts(time, postqc.burst_precision);
end
Expand All @@ -252,7 +232,7 @@
postqc.variables{k}.l = l;
postqc.variables{k}.r = r;

if has_burst
if ~isempty(burst_interval)
postqc.variables{k}.valid_burst_range = findBursts(postqc.variables{k}.time, postqc.burst_precision);
end

Expand All @@ -263,11 +243,67 @@
postqc.variables{k}.time = time;
postqc.variables{k}.l = 1;
postqc.variables{k}.r = numel(pqc_var.data);
if has_burst
if ~isempty(burst_interval)
postqc.variables{k}.valid_burst_range = postqc.burst_index_range;
end

end
end

end

function [is_burst_sampling,is_burst_metadata_valid,burst_interval] = get_burst_metadata(sample_data)
%function [is_burst_metadata_valid,burst_interval] = get_burst_metadata(sample_data)
%
% Discover if burst metadata is valid, by inspecting metadata at root level
% and at parser level (meta). If root level is valid, meta level is ignored.
% The burst_interval is also returned.
%
% Inputs:
%
% sample_data - the toolbox data struct.
%
% Outputs:
%
% is_burst_sampling - True if burst data is provided.
% is_burst_metadata_valid - True if both burst_duration and burst_interval are valid
% burst_interval - the burst interval value.
%
%
% author: hugo.oliveira@utas.edu.au
%


is_burst_sampling = false;
burst_interval = [];

valid_burst_duration = false;
valid_burst_interval = false;

if isfield(sample_data,'instrument_burst_duration') && ~isempty(sample_data.instrument_burst_duration)
is_burst_sampling = true;
burst_duration = sample_data.instrument_burst_duration;
valid_burst_duration = ~isnan(burst_duration) && burst_duration>0;
end

if ~valid_burst_duration && isfield(sample_data,'meta') && isfield(sample_data.meta,'instrument_burst_duration') && ~isempty(sample_data.meta.instrument_burst_duration)
is_burst_sampling = true;
burst_duration = sample_data.meta.instrument_burst_duration;
valid_burst_duration = ~isnan(burst_duration) && burst_duration>0;
end

if isfield(sample_data,'instrument_burst_interval') && ~isempty(sample_data.instrument_burst_interval)
is_burst_sampling = true;
burst_interval = sample_data.instrument_burst_interval;
valid_burst_interval = ~isnan(burst_interval) && burst_interval>0;
end

if ~valid_burst_interval && isfield(sample_data,'meta') && isfield(sample_data.meta,'instrument_burst_interval') && ~isempty(sample_data.meta.instrument_burst_interval)
is_burst_sampling = true;
burst_interval = sample_data.meta.instrument_burst_interval;
valid_burst_interval = ~isempty(burst_interval) && ~isnan(burst_interval) && burst_interval>0;
end

is_burst_metadata_valid = is_burst_sampling && valid_burst_duration && valid_burst_interval;

end
Binary file modified Java/ddb.jar
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion imosToolbox.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ function imosToolbox(auto, varargin)
%

% Set current toolbox version
toolboxVersion = ['2.6.8 - ' computer];
toolboxVersion = ['2.6.9 - ' computer];

if nargin == 0, auto = 'manual'; end

Expand Down
Binary file modified imosToolbox_Linux64.bin
View file
Binary file not shown.
Binary file modified imosToolbox_Win64.exe
View file
Binary file not shown.
62 changes: 49 additions & 13 deletions test/Parser/testnetcdfParse.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,32 +8,27 @@

methods (Test)

function test_no_error_reading_anmn(testCase, anmn_file)
function test_no_error_reading_anmn(~, anmn_file)
data = netcdfParse({anmn_file}, '');
assert(~isempty(data.meta.site_id))
assert(~isempty(data.time_coverage_start))
assert(~isempty(data.time_coverage_end))
assert(isnan(data.meta.instrument_sample_interval) || data.meta.instrument_sample_interval > 0)
assert((isnan(data.meta.instrument_burst_interval) || data.meta.instrument_burst_interval > 0) && (isnan(data.meta.instrument_burst_duration) || data.meta.instrument_burst_duration > 0))
assert_metadata(data);
assert_instrument_sample_interval(data);
assert_burst_sampling(data);
end

function test_no_error_reading_abos(~, abos_file)
data = netcdfParse({abos_file}, '');
assert(~isempty(data.meta.site_id))
assert(~isempty(data.time_coverage_start))
assert(~isempty(data.time_coverage_end))
assert(isnan(data.meta.instrument_sample_interval) || data.meta.instrument_sample_interval > 0)
assert((isnan(data.meta.instrument_burst_interval) || data.meta.instrument_burst_interval > 0) && (isnan(data.meta.instrument_burst_duration) || data.meta.instrument_burst_duration > 0))
assert_metadata(data);
assert_instrument_sample_interval(data);
assert_burst_sampling(data);
end

end

end

function ycell = only_netcdf(xcell)
ycell = {};
ycell = cell(1,1000);
c = 0;

for k = 1:length(xcell)
item = xcell{k};

Expand All @@ -43,5 +38,46 @@ function test_no_error_reading_abos(~, abos_file)
end

end
ycell=ycell(1:c);
end

function assert_metadata(data)
got_site_id = isfield(data,'meta') && isfield(data.meta,'site_id');
assert(got_site_id);
assert_site_id = ~isempty(data.meta.site_id);
assert(assert_site_id);

got_time_coverage = isfield(data,'time_coverage_start') && isfield(data,'time_coverage_end');
assert(got_time_coverage);
assert_time_coverage = ~isempty(data.time_coverage_start) && ~isempty(data.time_coverage_end);
assert(assert_time_coverage);
end

function assert_instrument_sample_interval(data)
if isfield(data,'instrument_sample_interval')
if ~isempty(data.instrument_sample_interval) && isfield(data,'meta') && isfield(data.meta,'instrument_sample_interval')
assert(data.instrument_sample_interval == data.meta.instrument_sample_interval)
end
else
warning('%s missing instrument_sample_interval',data.toolbox_input_file)
end

end

function assert_burst_sampling(data)
is_burst = isfield(data,'instrument_burst_interval') || isfield(data,'instrument_burst_duration');
if is_burst
has_burst_interval_but_not_duration = isfield(data,'instrument_burst_interval') && ~isfield(data,'instrument_burst_duration');
has_burst_duration_but_not_interval = isfield(data,'instrument_burst_duration') && ~isfield(data,'instrument_burst_interval');
if has_burst_interval_but_not_duration || has_burst_duration_but_not_interval
warning('%s contains incomplete burst information',data.toolbox_input_file)
else
assert_burst_interval = (~isnan(data.instrument_burst_interval)) || (data.instrument_burst_interval > 0);
assert_burst_duration = (~isnan(data.instrument_burst_duration)) || (data.instrument_burst_duration > 0);
assert(assert_burst_interval && assert_burst_duration)
if isfield(data,'meta')
assert_burst_sampling(data.meta)
end
end
end
end
Loading

0 comments on commit 64d4289

Please sign in to comment.