File: siroperator.h

package info (click to toggle)
aoflagger 3.1.0-2
  • links: PTS, VCS
  • area: main
  • in suites: bookworm
  • size: 4,868 kB
  • sloc: cpp: 52,164; python: 152; sh: 60; makefile: 17
file content (296 lines) | stat: -rw-r--r-- 10,763 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
#ifndef SIROPERATOR_H
#define SIROPERATOR_H

#include "../../structures/mask2d.h"
#include "../../structures/types.h"
#include "../../structures/xyswappedmask2d.h"

/**
 * This class contains functions that implement an algorithm to dilate
 * a flag mask: the "scale-invariant rank (SIR) operator".
 * The amount of dilation is relative to the size of the flagged
 * areas in the input, hence it is scale invariant. This behaviour is very
 * effective for application after amplitude based RFI detection and is a step
 * in the default LOFAR flagging pipeline.
 *
 * The rule for this scale invariant dilation is as follows:
 * Consider the sequence w(y) of size N, where w(y) = 0 if sample y is flagged
 * and w(y) = 1 otherwise. If there exists a subsequence within w that includes
 * y and that has a flagged ratio of η or more, y will be flagged.
 *
 * Thus:
 * if Y1 and Y2 exists, such that \\sum_{y=Y1}^{Y2-1} w(y)  <  η (Y2 - Y1), flag
 * y.
 *
 * The algorithm will be applied both in time and in frequency direction, thus
 * w(y) can contain a slice through the time-frequency image in either
 * directions.
 *
 * The algorithm is described in Offringa, van de Gronde and Roerdink 2012
 * (A&A).
 *
 * @author A.R. Offringa
 */
class SIROperator {
 public:
  /**
   * This is the proof of concept, reference version of the O(N) algorithm. It
   * is fast, but OperateHorizontally() and OperateVertically() have been
   * optimized for operating on a mask directly, which is the common mode of
   * operation.
   *
   * It contains extra comments to explain the algorithm within the code.
   *
   * @param [in,out] flags The input array of flags to be dilated that will be
   * overwritten by the dilatation of itself.
   * @param [in] flagsSize Size of the @c flags array.
   * @param [in] eta The η parameter that specifies the minimum number of good
   * data that any subsequence should have (see class description for the
   * definition).
   */
  static void Operate(bool* flags, const unsigned flagsSize, num_t eta) {
    // The test for a sample to become flagged can be rewritten as
    //         \\sum_{y=Y1}^{Y2-1} ( η - w(y) ) >= 0.
    // With w(y) =      flags[y] : 0
    //                 !flags[y] : 1

    // Make an array in which flagged samples are η and unflagged samples are
    // η-1, such that we can test for \\sum_{y=Y1}^{Y2-1} values[y] >= 0
    std::unique_ptr<num_t[]> values(new num_t[flagsSize]);
    for (unsigned i = 0; i < flagsSize; ++i) {
      if (flags[i])
        values[i] = eta;
      else
        values[i] = eta - 1.0;
    }

    // For each x, we will now search for the largest sum of sequantial values
    // that contains x. If this sum is larger then 0, this value is part of a
    // sequence that exceeds the test.

    // Define W(x) = \\sum_{y=0}^{x-1} values[y], such that the largest sequence
    // containing x starts at the element after W(y) is minimal in the range 0
    // <= y <= x, and ends when W(y) is maximal in the range x < y < N.

    // Calculate these W's and minimum prefixes
    const unsigned wSize = flagsSize + 1;
    std::unique_ptr<num_t[]> w(new num_t[wSize]);
    w[0] = 0.0;
    unsigned currentMinIndex = 0;
    std::unique_ptr<unsigned[]> minIndices(new unsigned[wSize]);
    minIndices[0] = 0;
    for (unsigned i = 1; i != wSize; ++i) {
      w[i] = w[i - 1] + values[i - 1];

      if (w[i] < w[currentMinIndex]) {
        currentMinIndex = i;
      }
      minIndices[i] = currentMinIndex;
    }

    // Calculate the maximum suffixes
    unsigned currentMaxIndex = wSize - 1;
    std::unique_ptr<unsigned[]> maxIndices(new unsigned[wSize]);
    for (unsigned i = flagsSize - 1; i != 0; --i) {
      // We directly assign maxIndices[i] to the max index over
      // all indices *higher* than i, since maxIndices[i] is
      // not allowed to be i (maxIndices[i] = max i: x < i < N).
      maxIndices[i] = currentMaxIndex;

      if (w[i] > w[currentMaxIndex]) {
        currentMaxIndex = i;
      }
    }
    maxIndices[0] = currentMaxIndex;

    // See if max sequence exceeds limit.
    for (unsigned i = 0; i != flagsSize; ++i) {
      const num_t maxW = w[maxIndices[i]] - w[minIndices[i]];
      flags[i] = (maxW >= 0.0);
    }
  }

  /**
   * Performs a horizontal dilation directly on a mask. Algorithm is equal to
   * Operate().
   *
   * @param [in,out] mask The input flag mask to be dilated.
   * @param [in] eta The η parameter that specifies the minimum number of good
   * data that any subsequence should have.
   */
  static void OperateHorizontally(Mask2D& mask, num_t eta) {
    operateHorizontally<Mask2D>(mask, eta);
  }

  /**
   * Performs a horizontal dilation directly on a mask, with missing value.
   * Missing values are values for which it is not know they should have
   * been flagged. An example is when the correlator has flagged
   * samples: these should be excluded from RFI detection, but they might
   * or might not contain RFI. The way this is implemented is by concatenating
   * all non-missing samples in a row, and performing the algorithm on that.
   *
   * @param [in,out] mask The input flag mask to be dilated.
   * @param [in] missing Flag mask that identifies missing values.
   * @param [in] eta The η parameter that specifies the minimum number of good
   * data that any subsequence should have.
   */
  static void OperateHorizontallyMissing(Mask2D& mask, const Mask2D& missing,
                                         num_t eta) {
    operateHorizontallyMissing<Mask2D>(mask, missing, eta);
  }

  /**
   * Performs a vertical dilation directly on a mask. Algorithm is equal to
   * Operate().
   *
   * @param [in,out] mask The input flag mask to be dilated.
   * @param [in] eta The η parameter that specifies the minimum number of good
   * data that any subsequence should have.
   */
  static void OperateVertically(Mask2D& mask, num_t eta) {
    XYSwappedMask2D<Mask2D> swappedMask(mask);
    operateHorizontally<XYSwappedMask2D<Mask2D>>(swappedMask, eta);
  }

  /**
   * Performs a vertical dilation directly on a mask, with missing value.
   * Identical to @ref OperateHorizontallyMissing(), but then vertically.
   *
   * @param [in,out] mask The input flag mask to be dilated.
   * @param [in] missing Flag mask that identifies missing values.
   * @param [in] eta The η parameter that specifies the minimum number of good
   * data that any subsequence should have.
   */
  static void OperateVerticallyMissing(Mask2D& mask, const Mask2D& missing,
                                       num_t eta) {
    XYSwappedMask2D<Mask2D> swappedMask(mask);
    XYSwappedMask2D<const Mask2D> swappedMissing(missing);
    operateHorizontallyMissing(swappedMask, swappedMissing, eta);
  }

 private:
  SIROperator() = delete;

  /**
   * Performs a horizontal dilation directly on a mask. Algorithm is equal to
   * Operate(). This is the implementation.
   *
   * @param [in,out] mask The input flag mask to be dilated.
   * @param [in] eta The η parameter that specifies the minimum number of good
   * data that any subsequence should have.
   */
  template <typename MaskLike>
  static void operateHorizontally(MaskLike& mask, num_t eta) {
    const unsigned width = mask.Width(), wSize = width + 1;
    std::unique_ptr<num_t[]> values(new num_t[width]), w(new num_t[wSize]);
    std::unique_ptr<unsigned[]> minIndices(new unsigned[wSize]),
        maxIndices(new unsigned[wSize]);

    for (unsigned row = 0; row < mask.Height(); ++row) {
      for (unsigned i = 0; i < width; ++i) {
        if (mask.Value(i, row))
          values[i] = eta;
        else
          values[i] = eta - 1.0;
      }

      w[0] = 0.0;
      unsigned currentMinIndex = 0;
      minIndices[0] = 0;
      for (unsigned i = 1; i != wSize; ++i) {
        w[i] = w[i - 1] + values[i - 1];

        if (w[i] < w[currentMinIndex]) {
          currentMinIndex = i;
        }
        minIndices[i] = currentMinIndex;
      }

      // Calculate the maximum suffixes
      unsigned currentMaxIndex = wSize - 1;
      for (unsigned i = width - 1; i != 0; --i) {
        maxIndices[i] = currentMaxIndex;
        if (w[i] > w[currentMaxIndex]) {
          currentMaxIndex = i;
        }
      }
      maxIndices[0] = currentMaxIndex;

      // See if max sequence exceeds limit.
      for (unsigned i = 0; i != width; ++i) {
        const num_t maxW = w[maxIndices[i]] - w[minIndices[i]];
        mask.SetValue(i, row, (maxW >= 0.0));
      }
    }
  }

  /**
   * Performs a horizontal dilation directly on a mask with missing values.
   * This is the implementation.
   *
   * @param [in,out] mask The input flag mask to be dilated.
   * @param [in] missing Flag mask that identifies missing values.
   * @param [in] eta The η parameter that specifies the minimum number of good
   * data that any subsequence should have.
   */
  template <typename MaskLikeA, typename MaskLikeB>
  static void operateHorizontallyMissing(MaskLikeA& mask,
                                         const MaskLikeB& missing, num_t eta) {
    const unsigned width = mask.Width(), maxWSize = width + 1;
    std::unique_ptr<num_t[]> values(new num_t[width]), w(new num_t[maxWSize]);
    std::unique_ptr<unsigned[]> minIndices(new unsigned[maxWSize]),
        maxIndices(new unsigned[maxWSize]);

    for (unsigned row = 0; row < mask.Height(); ++row) {
      unsigned nAvailable = 0;
      for (unsigned i = 0; i < width; ++i) {
        if (!missing.Value(i, row)) {
          if (mask.Value(i, row))
            values[nAvailable] = eta;
          else
            values[nAvailable] = eta - 1.0;
          ++nAvailable;
        }
      }

      if (nAvailable != 0) {
        unsigned wSize = nAvailable + 1;
        w[0] = 0.0;
        unsigned currentMinIndex = 0;
        minIndices[0] = 0;
        for (unsigned i = 1; i != wSize; ++i) {
          w[i] = w[i - 1] + values[i - 1];

          if (w[i] < w[currentMinIndex]) {
            currentMinIndex = i;
          }
          minIndices[i] = currentMinIndex;
        }

        // Calculate the maximum suffixes
        unsigned currentMaxIndex = wSize - 1;
        for (unsigned i = nAvailable - 1; i != 0; --i) {
          maxIndices[i] = currentMaxIndex;
          if (w[i] > w[currentMaxIndex]) {
            currentMaxIndex = i;
          }
        }
        maxIndices[0] = currentMaxIndex;

        // See if max sequence exceeds limit.
        nAvailable = 0;
        for (unsigned i = 0; i != width; ++i) {
          if (!missing.Value(i, row)) {
            const num_t maxW =
                w[maxIndices[nAvailable]] - w[minIndices[nAvailable]];
            mask.SetValue(i, row, (maxW >= 0.0));
            ++nAvailable;
          }
        }
      }
    }
  }
};

#endif