File: README.md

package info (click to toggle)
node-unique-stream 2.2.1-1
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, buster, sid, stretch
  • size: 96 kB
  • ctags: 8
  • sloc: makefile: 4; sh: 2
file content (133 lines) | stat: -rw-r--r-- 3,514 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
# unique-stream

node.js through stream that emits a unique stream of objects based on criteria

[![Build Status](https://travis-ci.org/eugeneware/unique-stream.svg?branch=master)](https://travis-ci.org/eugeneware/unique-stream)
[![Coverage Status](https://coveralls.io/repos/eugeneware/unique-stream/badge.svg?branch=master&service=github)](https://coveralls.io/github/eugeneware/unique-stream?branch=master)

## Installation

Install via [npm](https://www.npmjs.com/):

```
$ npm install unique-stream
```

## Examples

### Dedupe a ReadStream based on JSON.stringify:

``` js
var unique = require('unique-stream')
  , Stream = require('stream');

// return a stream of 3 identical objects
function makeStreamOfObjects() {
  var s = new Stream;
  s.readable = true;
  var count = 3;
  for (var i = 0; i < 3; i++) {
    setImmediate(function () {
      s.emit('data', { name: 'Bob', number: 123 });
      --count || end();
    });
  }

  function end() {
    s.emit('end');
  }

  return s;
}

// Will only print out one object as the rest are dupes. (Uses JSON.stringify)
makeStreamOfObjects()
  .pipe(unique())
  .on('data', console.log);

```

### Dedupe a ReadStream based on an object property:

``` js
// Use name as the key field to dedupe on. Will only print one object
makeStreamOfObjects()
  .pipe(unique('name'))
  .on('data', console.log);
```

### Dedupe a ReadStream based on a custom function:

``` js
// Use a custom function to dedupe on. Use the 'number' field. Will only print one object.
makeStreamOfObjects()
  .pipe(function (data) {
    return data.number;
  })
  .on('data', console.log);
```

## Dedupe multiple streams

The reason I wrote this was to dedupe multiple object streams:

``` js
var aggregator = unique();

// Stream 1
makeStreamOfObjects()
  .pipe(aggregator);

// Stream 2
makeStreamOfObjects()
  .pipe(aggregator);

// Stream 3
makeStreamOfObjects()
  .pipe(aggregator);

aggregator.on('data', console.log);
```

## Use a custom store to record keys that have been encountered

By default a set is used to store keys encountered so far, in order to check new ones for
uniqueness. You can supply your own store instead, providing it supports the add(key) and 
has(key) methods. This could allow you to use a persistant store so that already encountered
objects are not re-streamed when node is reloaded.

``` js
var keyStore = {
  store: {},

  add: function(key) {
    this.store[key] = true;
  },

  has: function(key) {
    return this.store[key] !== undefined;
  }
};
    
makeStreamOfObjects()
  .pipe(unique('name', keyStore))
  .on('data', console.log);
```

## Contributing

unique-stream is an **OPEN Open Source Project**. This means that:

> Individuals making significant and valuable contributions are given commit-access to the project to contribute as they see fit. This project is more like an open wiki than a standard guarded open source project.

See the [CONTRIBUTING.md](https://github.com/eugeneware/unique-stream/blob/master/CONTRIBUTING.md) file for more details.

### Contributors

unique-stream is only possible due to the excellent work of the following contributors:

<table><tbody>
<tr><th align="left">Eugene Ware</th><td><a href="https://github.com/eugeneware">GitHub/eugeneware</a></td></tr>
<tr><th align="left">Craig Ambrose</th><td><a href="https://github.com/craigambrose">GitHub/craigambrose</a></td></tr>
<tr><th align="left">Shinnosuke Watanabe</th><td><a href="https://github.com/shinnn">GitHub/shinnn</a></td></tr>
</tbody></table>